framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,1,0.011045332998037338
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,7,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,7,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,3,0.01119999960064888
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,1,0.011066666493813196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,15,0.010821333775917688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,15,0.010768000036478043
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,31,0.01062400018175443
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,31,0.010928000013033548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,63,0.010954666882753372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,63,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,127,0.010965333630641302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,127,0.011168000598748526
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,255,0.01109333336353302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,255,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,511,0.014378666877746582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,511,0.01301866645614306
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,1023,0.0145066666106383
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,3,0.011370666325092316
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,1023,0.013946666071812311
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,2047,0.029520000020662945
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,2047,0.017103999853134155
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,4095,0.031386665999889374
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,4095,0.047050664822260536
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,1,0.012703999876976013
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,1,0.011007999380429586
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,3,0.012730666746695837
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,3,0.011317333827416102
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,7,0.012624000509579977
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,7,0.011194666226704916
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,15,0.013141332815090815
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,31,0.01118933285276095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,31,0.01239466667175293
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,15,0.011237333218256632
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,63,0.013301332791646322
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,63,0.010885333021481832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,127,0.012826666235923767
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,127,0.011120000233252844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,255,0.012773333738247553
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,255,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,511,0.014805333067973455
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,511,0.013077333569526672
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,1023,0.02956266701221466
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,1023,0.01515199989080429
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,2047,0.047728002071380615
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,2047,0.03387733300526937
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,4095,0.08152533570925395
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,1,0.010778666784365972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,4095,0.04850133260091146
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,1,0.01108266661564509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,3,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,3,0.010602666685978571
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,7,0.010847999403874079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,7,0.010837333897749582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,15,0.010661333799362183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,15,0.011055999745925268
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,31,0.010064000263810158
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,31,0.011018666128317514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,63,0.011087999989589056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,63,0.010533332824707031
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,127,0.010618666807810465
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,127,0.010751999914646149
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,255,0.010778666784365972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,255,0.011146667102972666
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,511,0.010869332899649939
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,511,0.011136000355084738
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,1023,0.01121066634853681
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,1023,0.011136000355084738
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,2047,0.010981333752473196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,2047,0.01097600037852923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,4095,0.013077333569526672
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,4095,0.012752000242471695
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,8191,0.01721599946419398
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,16383,0.019413333386182785
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,8191,0.01586666703224182
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,16383,0.01934933289885521
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,1,0.010746666540702185
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,1,0.009125333279371262
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,3,0.010874666273593903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,3,0.009093333035707474
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,7,0.008821333448092142
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,7,0.01028266673286756
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,15,0.009914666414260864
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,15,0.00902399979531765
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,31,0.01055466632048289
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,31,0.008832000195980072
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,63,0.010869332899649939
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,63,0.010570666442314783
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,127,0.008874666566650072
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,127,0.010634666929642359
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,255,0.010666667173306147
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,255,0.0099093330403169
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,511,0.010618666807810465
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,511,0.01081066702802976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,1023,0.011029332876205444
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,1023,0.010869332899649939
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,2047,0.010821333775917688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,2047,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,4095,0.013104000439246496
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,4095,0.013242666920026144
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,8191,0.015178666760524115
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,8191,0.015141333142916361
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,16383,0.017221332838137943
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,1,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,16383,0.01699200024207433
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,1,0.01071999967098236
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,3,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,3,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,7,0.008949333180983862
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,7,0.01098666712641716
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,15,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,15,0.01108266661564509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,31,0.01102399950226148
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,31,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,63,0.00919999989370505
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,63,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,127,0.01071999967098236
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,127,0.00996800015370051
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,255,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,255,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,511,0.011152000476916632
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,1023,0.01119999960064888
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,511,0.010885333021481832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,1023,0.011034666250149408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,2047,0.01129066695769628
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,2047,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,8191,0.01607999950647354
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,4095,0.015306666493415833
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,4095,0.015077333897352219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,8191,0.015274666249752045
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,16383,0.01732800031701724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,16383,0.019194666296243668
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,1,0.01091733326514562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,1,0.009322666873534521
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,3,0.009509333098928133
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,3,0.010677333921194077
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,7,0.010602666685978571
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,7,0.009237333511312803
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,15,0.010928000013033548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,15,0.010090666512648264
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,31,0.00901333304742972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,31,0.009594666461149851
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,63,0.009674666449427605
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,127,0.00996800015370051
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,63,0.011205332974592844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,127,0.008879999940594038
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,255,0.010645333677530289
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,511,0.010794666906197866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,255,0.00980266680320104
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,511,0.010666667173306147
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,1023,0.01091733326514562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,1023,0.01062400018175443
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,2047,0.01314666618903478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,2047,0.012762666990359625
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,4095,0.013114667187134424
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,4095,0.014245333770910898
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,8191,0.016997333616018295
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,8191,0.01515199989080429
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,16383,0.017045332739750545
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,16383,0.03152533372243246
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,1,0.012661332885424295
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,1,0.01301866645614306
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,3,0.012885333349307379
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,3,0.013162666310866674
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,7,0.0129120002190272
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,7,0.012906666845083237
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,15,0.013045333325862885
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,15,0.012917333592971167
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,31,0.013104000439246496
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,31,0.01301866645614306
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,63,0.0129120002190272
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,63,0.012885333349307379
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,127,0.012842666357755661
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,127,0.013072000195582708
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,255,0.012890666723251343
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,255,0.012863999853531519
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,511,0.01526933287580808
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,511,0.014869333555301031
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,1023,0.029653333127498627
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,1023,0.017263999829689663
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,2047,0.04823466638724009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,2047,0.03183466692765554
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,1,0.015018666783968607
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,1,0.013173333058754602
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,3,0.013088000317414602
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,3,0.014906667172908783
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,7,0.014607999473810196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,7,0.013248000293970108
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,15,0.013050666699806849
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,15,0.015125333021084467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,31,0.015141333142916361
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,31,0.014917333920796713
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,63,0.014938666174809137
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,63,0.013082666943470636
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,127,0.014959999670584997
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,127,0.014826666563749313
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,255,0.014794666320085526
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,255,0.013082666943470636
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,511,0.017050666113694508
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,511,0.03088533381621043
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,1023,0.031343999008337654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,1023,0.0484746644894282
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,1,0.010682666053374609
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,1,0.0107893335322539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,2047,0.08474133412043254
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,2047,0.0498879998922348
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,3,0.010538666198650995
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,3,0.010965333630641302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,7,0.009183999771873156
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,7,0.010869332899649939
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,15,0.010970667004585266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,15,0.010805333654085795
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,31,0.010677333921194077
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,31,0.010608000059922537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,63,0.009898666913310686
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,63,0.010970667004585266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,127,0.009919999788204828
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,127,0.01080000028014183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,255,0.010309333602587381
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,255,0.010629333555698395
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,511,0.011087999989589056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,511,0.011205332974592844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,1023,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,2047,0.01314666618903478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,1023,0.011039999624093374
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,2047,0.012997332960367203
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,4095,0.015146666516860327
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,4095,0.015146666516860327
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,8191,0.01703466723362605
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,8191,0.01748266691962878
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,16383,0.03282133241494497
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,16383,0.01916266605257988
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,1,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,1,0.010757333288590113
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,3,0.008992000172535578
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,3,0.01071999967098236
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,7,0.010842667271693548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,7,0.010608000059922537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,15,0.010138666878143946
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,31,0.010533332824707031
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,15,0.011050666371981302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,31,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,63,0.009557333464423815
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,63,0.010650667051474253
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,127,0.010239999741315842
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,127,0.010703999549150467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,255,0.010389333590865135
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,255,0.010698666175206503
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,511,0.011221333096424738
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,511,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,1023,0.01108266661564509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,1023,0.01081066702802976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,2047,0.011173332730929056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,2047,0.012896000097195307
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,4095,0.012917333592971167
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,4095,0.015082667271296183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,8191,0.016095999628305435
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,8191,0.029365333418051403
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,16383,0.04706666866938273
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,16383,0.030794667700926464
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,1,0.015402667224407196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,1,0.014959999670584997
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,3,0.015626666446526844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,3,0.014885333677132925
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,7,0.015306666493415833
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,7,0.015040000279744467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,15,0.015386667102575302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,15,0.014906667172908783
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,31,0.015061333775520325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,31,0.015930666277805965
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,63,0.015077333897352219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,63,0.014938666174809137
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,127,0.015263999501864115
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,127,0.015135999768972397
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,255,0.01534933348496755
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,255,0.015210667004187902
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,511,0.03148799886306127
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,511,0.01930133377512296
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,1023,0.0499893327554067
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,1023,0.03349333256483078
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,1,0.019109333554903667
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,1,0.017418666432301205
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,3,0.01889066646496455
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,3,0.01743999992807706
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,7,0.019199999670187633
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,7,0.01729600007335345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,15,0.018917333334684372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,15,0.01729600007335345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,31,0.01746133342385292
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,31,0.018992000569899876
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,63,0.018981333822011948
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,63,0.017360000560681026
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,127,0.017194667210181553
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,127,0.019237333287795384
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,255,0.029882666965325672
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,255,0.017184000462293625
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,511,0.03197333216667175
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,511,0.04806933303674062
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,1023,0.08264000217119853
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,1023,0.0499946673711141
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,1,0.02294933299223582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,1,0.019178666174411774
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,3,0.021429332594076794
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,3,0.019610666980346043
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,7,0.021210665504137676
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,7,0.019023999571800232
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,15,0.019567999988794327
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,31,0.021407999098300934
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,31,0.019088000059127808
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,15,0.02276266614596049
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,63,0.02309333284695943
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,63,0.019359999646743137
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,127,0.021386665602525074
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,127,0.019567999988794327
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,255,0.031498665610949196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,255,0.019551999866962433
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,1,0.027530667682488758
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,1,0.025594666600227356
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,7,0.027327999472618103
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,3,0.02827200045188268
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,3,0.025093334416548412
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,7,0.02534399926662445
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,15,0.027615999182065327
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,15,0.025242666403452556
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,31,0.028175999720891316
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,31,0.025279998779296875
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,63,0.02531733363866806
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,63,0.02731200059254964
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,127,0.03350399931271871
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,127,0.025450666745503742
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,255,0.03312533348798752
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,255,0.0495306650797526
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,1,0.03585066646337509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,1,0.03120533376932144
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,3,0.0355679988861084
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,3,0.03146133323510488
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,7,0.0367999995748202
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,7,0.03163733333349228
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,15,0.03730133424202601
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,15,0.03151999910672506
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,31,0.036506667733192444
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,31,0.03154666721820831
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,63,0.035818666219711304
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,63,0.0316746657093366
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,127,0.039546666045983635
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,1,0.05009600023428599
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,127,0.03148266673088074
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,1,0.04374399781227112
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,3,0.05212800204753876
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,3,0.04367466767628988
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,7,0.04178133110205332
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,7,0.050437331199645996
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,15,0.05150933563709259
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,15,0.04353600243727366
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,31,0.05020266771316528
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,31,0.04390933116277059
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,63,0.05206400156021118
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,63,0.04260266820589701
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,1,0.010746666540702185
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,3,0.010762666662534079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,1,0.010586666564146677
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,127,0.05620799958705902
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,127,0.04663999875386556
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,3,0.01062400018175443
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,7,0.00960533320903778
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,15,0.010821333775917688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,15,0.010559999694426855
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,31,0.010751999914646149
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,63,0.00955200009047985
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,7,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,31,0.010527999450763067
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,63,0.010698666175206503
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,127,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,127,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,255,0.01071999967098236
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,255,0.011007999380429586
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,511,0.011034666250149408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,511,0.011034666250149408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,1023,0.01116266722480456
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,1023,0.012879999975363413
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,2047,0.01292266696691513
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,2047,0.013189333180586496
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,4095,0.015194666882356008
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,4095,0.015034666905800501
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,8191,0.029792000850041706
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,8191,0.0174346665541331
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,16383,0.04794666667779287
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,16383,0.03159466634194056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,1,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,1,0.010911999891201654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,3,0.010672000547250112
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,3,0.010656000425418219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,7,0.010938666760921478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,7,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,15,0.01071999967098236
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,15,0.01097600037852923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,31,0.010735999792814255
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,31,0.011018666128317514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,63,0.010666667173306147
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,63,0.010703999549150467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,127,0.011034666250149408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,127,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,255,0.010821333775917688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,255,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,511,0.010821333775917688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,511,0.011253333340088526
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,1023,0.012085333466529846
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,1023,0.01099733387430509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,2047,0.013013333082199097
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,2047,0.012991999586423239
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,4095,0.02937600016593933
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,4095,0.01685333376129468
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,8191,0.030746666093667347
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,8191,0.04693866769472758
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,16383,0.08075733482837677
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,16383,0.047728002071380615
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,float16,1,0.06625600159168243
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,1,0.055013333757718406
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,float16,3,0.06790933509667714
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,3,0.05598400036493937
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,float16,7,0.06651199857393901
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,float16,15,0.06791999936103821
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,7,0.055642664432525635
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,15,0.05487466851870219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,31,0.05455466608206431
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,float16,31,0.06637333333492279
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,float16,63,0.06754133105278015
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,63,0.05449066559473673
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,float16,1,0.09091732899347942
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,1,0.0809333324432373
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,float16,3,0.091648002465566
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,3,0.08082666496435802
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,float16,7,0.09160533547401428
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,7,0.0806879997253418
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,float16,15,0.09265066186587016
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,15,0.08045866588751475
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,float16,31,0.09103999535242717
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,31,0.08059733112653096
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,float16,63,0.09165333708127339
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,63,0.0813973347345988
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,float16,3,0.12395733594894409
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,float16,1,0.12388799587885539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,1,0.1018453339735667
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,3,0.10325866937637329
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,float16,7,0.1242026686668396
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,7,0.10206400354703267
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,15,0.10334933797518413
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,float16,15,0.12219732999801636
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,float16,31,0.12405332922935486
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,31,0.10172800223032634
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,float16,1,0.16709333658218384
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,1,0.1483680009841919
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,3,0.14829333623250326
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,float16,3,0.17074666420618692
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,float16,7,0.16701332728068033
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,7,0.1490293343861898
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,float16,15,0.17114667097727457
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,1,0.01108266661564509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,15,0.15026666720708212
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,1,0.010741333166758219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,3,0.01099733387430509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,3,0.010757333288590113
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,7,0.011055999745925268
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,7,0.01119999960064888
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,15,0.010944000134865442
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,15,0.01097600037852923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,31,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,31,0.011002667248249054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,63,0.010656000425418219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,63,0.01051733394463857
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,127,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,127,0.010816000401973724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,float16,31,0.1688213348388672
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,31,0.14854400356610617
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,255,0.01101333275437355
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,255,0.01081066702802976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,511,0.011087999989589056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,1023,0.013151999562978745
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,2047,0.014709333578745524
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,511,0.01157333329319954
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,1023,0.013056000073750814
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,2047,0.015018666783968607
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,4095,0.02993600070476532
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,4095,0.017221332838137943
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,8191,0.047770669062932335
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,8191,0.03141333411137263
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,1,0.010693332801262537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,1,0.010687999427318573
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,3,0.010842667271693548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,3,0.010938666760921478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,7,0.010847999403874079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,7,0.011045332998037338
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,15,0.010677333921194077
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,15,0.011125333607196808
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,31,0.010832000523805618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,31,0.010618666807810465
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,63,0.010938666760921478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,63,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,127,0.010874666273593903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,127,0.01121066634853681
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,255,0.011050666371981302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,255,0.01109333336353302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,511,0.01108266661564509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,511,0.011173332730929056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,1023,0.013013333082199097
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,1023,0.012693333129088083
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,2047,0.02962133288383484
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,2047,0.015098666151364645
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,4095,0.04597333570321401
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,4095,0.03075733284155528
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,8191,0.0812799980243047
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,8191,0.04749333361784617
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,1,0.010981333752473196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,3,0.011061333119869232
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,1,0.010757333288590113
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,3,0.011077333241701126
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,7,0.0107893335322539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,7,0.011055999745925268
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,15,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,15,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,31,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,31,0.011226666470368704
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,63,0.01102399950226148
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,63,0.01097600037852923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,127,0.011365332951148352
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,127,0.011530666301647821
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,255,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,255,0.011007999380429586
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,511,0.012661332885424295
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,1023,0.01393066719174385
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,511,0.01313599944114685
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,1023,0.013034666577974955
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,2047,0.03001066545645396
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,2047,0.01693333312869072
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,1,0.011087999989589056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,4095,0.031514666974544525
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,4095,0.0480373352766037
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,1,0.011045332998037338
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,3,0.012736000120639801
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,3,0.011050666371981302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,7,0.011194666226704916
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,7,0.01202133297920227
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,15,0.01293333371480306
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,15,0.011253333340088526
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,31,0.010965333630641302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,31,0.01138666644692421
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,63,0.012752000242471695
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,63,0.011029332876205444
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,127,0.011125333607196808
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,127,0.011461333682139715
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,255,0.013061333447694778
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,255,0.012928000340859095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,511,0.012986666212479273
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,511,0.012986666212479273
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,1023,0.029680001238981884
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,1023,0.015077333897352219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,2047,0.0470773329337438
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,2047,0.0313226655125618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,1,0.008767999708652496
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,1,0.010682666053374609
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,4095,0.0817386656999588
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,4095,0.048570667703946434
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,3,0.008938666433095932
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,3,0.010618666807810465
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,7,0.010538666198650995
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,7,0.009984000275532404
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,15,0.00895999992887179
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,15,0.010751999914646149
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,31,0.0107893335322539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,63,0.0107893335322539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,31,0.010634666929642359
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,63,0.011039999624093374
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,127,0.011045332998037338
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,127,0.01080000028014183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,255,0.00996800015370051
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,255,0.011055999745925268
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,511,0.011130666981140772
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,511,0.010992000500361124
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,1023,0.010794666906197866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,1023,0.011077333241701126
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,2047,0.011114666859308878
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,2047,0.01116266722480456
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,4095,0.011370666325092316
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,4095,0.012815999488035837
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,8191,0.01722666621208191
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,8191,0.017125333348910015
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,16383,0.018826667219400406
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,16383,0.01923199991385142
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,1,0.010847999403874079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,1,0.010981333752473196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,3,0.010794666906197866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,3,0.0102613332370917
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,7,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,7,0.010992000500361124
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,15,0.010677333921194077
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,31,0.008863999818762144
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,31,0.011087999989589056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,15,0.008997333546479544
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,63,0.0103946669648091
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,63,0.00891733355820179
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,127,0.00891733355820179
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,127,0.010784000158309937
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,255,0.01055466632048289
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,255,0.00949866697192192
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,511,0.01073066641887029
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,511,0.011152000476916632
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,1023,0.01062400018175443
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,1023,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,2047,0.011071999867757162
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,2047,0.011077333241701126
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,4095,0.013242666920026144
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,4095,0.013573333621025085
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,8191,0.01523200049996376
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,8191,0.015168000012636185
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,16383,0.017279999951521557
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,1,0.010816000401973724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,16383,0.017050666113694508
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,1,0.01073066641887029
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,3,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,3,0.010981333752473196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,7,0.010762666662534079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,7,0.01080000028014183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,15,0.01101333275437355
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,15,0.010837333897749582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,31,0.010725333044926325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,31,0.011045332998037338
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,63,0.010618666807810465
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,63,0.011061333119869232
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,127,0.010944000134865442
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,127,0.011050666371981302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,255,0.010570666442314783
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,255,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,511,0.011034666250149408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,511,0.01126933346192042
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,1023,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,1023,0.011610666910807291
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,2047,0.011215999722480774
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,4095,0.01515199989080429
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,2047,0.011727999895811081
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,4095,0.015141333142916361
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,8191,0.017360000560681026
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,8191,0.015493333339691162
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,16383,0.019050666441520054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,16383,0.01897066707412402
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,1,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,1,0.008922666932145754
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,3,0.010709332923094431
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,3,0.009312000125646591
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,7,0.009237333511312803
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,7,0.010586666564146677
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,15,0.010640000303586325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,31,0.01062400018175443
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,15,0.011034666250149408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,31,0.009141333401203156
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,63,0.00997866690158844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,63,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,127,0.009648000200589498
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,127,0.011098666737476984
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,255,0.01080000028014183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,255,0.010869332899649939
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,511,0.011066666493813196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,511,0.011050666371981302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,1023,0.011130666981140772
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,1023,0.010911999891201654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,2047,0.012784000486135483
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,2047,0.01301866645614306
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,4095,0.014853333433469137
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,4095,0.014975999792416891
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,8191,0.015205333630243937
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,8191,0.015119999647140503
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,16383,0.017877332866191864
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,16383,0.03164266546567281
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,1,0.013125333935022354
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,1,0.013183999806642532
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,3,0.013301332791646322
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,3,0.013141332815090815
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,7,0.012949333836634954
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,7,0.013141332815090815
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,15,0.012975999464591345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,15,0.013269333789745966
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,31,0.01303999995191892
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,63,0.013264000415802002
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,31,0.012896000097195307
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,63,0.013157332936922709
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,127,0.012928000340859095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,127,0.013002666334311167
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,255,0.013167999684810638
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,255,0.012997332960367203
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,511,0.015114666273196539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,511,0.014815999815861383
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,1023,0.017018667111794155
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,1023,0.030879999200503033
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,2047,0.048783997694651283
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,2047,0.03235200047492981
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,1,0.01516266663869222
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,1,0.014671999961137772
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,3,0.013141332815090815
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,3,0.014943999548753103
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,7,0.012954667210578918
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,7,0.014949332922697067
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,15,0.013066666821638743
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,15,0.015050667027632395
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,31,0.01498666654030482
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,31,0.012997332960367203
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,63,0.014858666807413101
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,63,0.013269333789745966
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,127,0.013237333546082178
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,127,0.015082667271296183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,255,0.01492799942692121
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,255,0.013221333424250284
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,511,0.029738667110602062
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,511,0.017157333592573803
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,1023,0.050554667909940086
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,1,0.011194666226704916
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,1023,0.03151999910672506
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,2047,0.08399466673533122
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,1,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,3,0.011055999745925268
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,3,0.011034666250149408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,7,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,2047,0.05004799862702688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,7,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,15,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,15,0.010768000036478043
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,31,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,63,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,31,0.011045332998037338
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,63,0.010645333677530289
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,127,0.011039999624093374
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,127,0.011007999380429586
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,255,0.010634666929642359
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,255,0.011157333850860596
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,511,0.01116266722480456
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,511,0.010992000500361124
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,1023,0.011205332974592844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,1023,0.011642667154471079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,2047,0.013061333447694778
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,2047,0.01301866645614306
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,4095,0.015274666249752045
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,8191,0.01704000060757001
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,4095,0.015365333606799444
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,8191,0.01714133347074191
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,16383,0.03330666571855545
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,16383,0.019071999937295914
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,1,0.009269333134094873
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,1,0.010970667004585266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,3,0.010255999863147736
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,3,0.00914666677514712
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,7,0.00902399979531765
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,7,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,15,0.010106666634480158
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,15,0.009008000294367472
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,31,0.009226666763424873
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,31,0.010832000523805618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,63,0.009232000137368837
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,63,0.010672000547250112
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,127,0.010794666906197866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,127,0.009296000003814697
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,255,0.010453333457310995
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,255,0.010687999427318573
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,511,0.011141333729028702
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,511,0.01139733319481214
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,1023,0.011226666470368704
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,1023,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,2047,0.012533333152532578
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,2047,0.012975999464591345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,4095,0.013167999684810638
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,4095,0.014858666807413101
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,8191,0.01571200042963028
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,16383,0.03162133445342382
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,8191,0.029631999631722767
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,16383,0.04674666623274485
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,1,0.015247999380032221
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,1,0.015184000134468079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,3,0.014997333288192749
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,3,0.01661866654952367
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,7,0.015311999867359797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,7,0.015247999380032221
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,15,0.01516266663869222
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,15,0.015205333630243937
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,31,0.015018666783968607
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,31,0.014842666685581207
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,63,0.017375999440749485
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,63,0.015098666151364645
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,127,0.015376000354687372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,127,0.014864000181357065
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,255,0.015557333827018738
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,255,0.015146666516860327
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,511,0.03150933235883713
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,511,0.019141333798567455
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,1023,0.049360002080599465
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,1023,0.03368533402681351
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,1,0.01720533271630605
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,1,0.01913600042462349
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,3,0.01916266605257988
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,3,0.017386666188637417
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,7,0.019098666807015736
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,7,0.017194667210181553
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,15,0.019306667149066925
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,15,0.017557332913080852
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,31,0.01701333373785019
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,31,0.01912533367673556
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,63,0.019386666516462963
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,63,0.017445333302021027
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,127,0.01916266605257988
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,127,0.017338667064905167
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,255,0.01718933383623759
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,255,0.029477333029111225
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,511,0.048570667703946434
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,1023,0.04942933221658071
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,1023,0.08262399832407634
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,511,0.03154666721820831
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,1,0.023567999402681988
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,1,0.019088000059127808
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,3,0.02143999934196472
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,3,0.019343999524911244
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,7,0.019317333896954853
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,7,0.023370665808518726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,15,0.02120000123977661
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,15,0.019386666516462963
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,31,0.02146666745344798
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,31,0.01932799940307935
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,63,0.021205333371957142
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,63,0.01932266727089882
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,127,0.023269332945346832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,127,0.019440000255902607
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,255,0.03136533250411352
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,255,0.020527999848127365
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,1,0.025578667720158894
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,1,0.02757333219051361
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,3,0.027632000545660656
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,3,0.025461333493391674
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,7,0.025146665672461193
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,7,0.027471999327341717
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,15,0.027664000789324444
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,15,0.025407999753952026
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,31,0.02555199960867564
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,31,0.027797333896160126
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,63,0.027317332724730175
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,63,0.025434667865435284
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,127,0.025477332373460133
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,127,0.03349866718053818
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,255,0.048826664686203
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,255,0.03263466556866964
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,1,0.031658666829268135
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,1,0.03566933423280716
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,3,0.035504000882307686
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,3,0.03161066770553589
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,7,0.03128000100453695
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,7,0.03643733263015747
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,15,0.035775999228159584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,15,0.031311998764673867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,31,0.0359199990828832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,31,0.03130666663249334
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,63,0.036090667049090065
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,63,0.03126933425664902
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,127,0.03975466638803482
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,127,0.03169599920511246
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,1,0.05097599824269613
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,1,0.0428959975639979
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,3,0.050373335679372154
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,3,0.04367466767628988
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,7,0.050213331977526345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,7,0.04383466641108195
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,15,0.05048533280690511
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,15,0.043738668163617454
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,31,0.05022933085759481
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,31,0.0436160018046697
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,63,0.050154666105906166
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,63,0.0439626673857371
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,1,0.01098666712641716
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,1,0.010970667004585266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,3,0.011098666737476984
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,3,0.01116266722480456
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,127,0.0562666654586792
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,7,0.011098666737476984
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,7,0.01073066641887029
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,15,0.011018666128317514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,31,0.0124746672809124
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,15,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,31,0.011029332876205444
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,63,0.010512000570694605
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,63,0.011055999745925268
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,127,0.011002667248249054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,127,0.01091733326514562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,255,0.009818666925032934
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,127,0.0470719983180364
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,255,0.01097600037852923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,511,0.011152000476916632
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,511,0.010981333752473196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,1023,0.011194666226704916
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,2047,0.013013333082199097
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,2047,0.012981332838535309
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,1023,0.012960000584522883
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,4095,0.015050667027632395
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,4095,0.014874666929244995
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,8191,0.029872000217437744
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,8191,0.018272000054518383
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,16383,0.048101335763931274
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,16383,0.0317493329445521
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,1,0.010992000500361124
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,1,0.010805333654085795
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,3,0.010735999792814255
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,3,0.010992000500361124
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,7,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,7,0.01073066641887029
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,15,0.010842667271693548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,15,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,31,0.010608000059922537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,31,0.010885333021481832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,63,0.010661333799362183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,63,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,127,0.009984000275532404
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,127,0.010714666297038397
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,255,0.01097600037852923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,255,0.010869332899649939
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,511,0.011215999722480774
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,511,0.011050666371981302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,1023,0.011871999750534693
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,1023,0.012831999609867731
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,2047,0.012975999464591345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,2047,0.013951999445756277
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,4095,0.029343999922275543
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,4095,0.015040000279744467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,8191,0.047226667404174805
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,8191,0.03140799949566523
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,16383,0.08067200084527333
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,16383,0.04797866443792979
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,1,0.05392000079154968
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,float16,1,0.0663679987192154
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,3,0.05417599777380625
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,float16,3,0.06646933158238728
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,float16,7,0.06649066507816315
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,7,0.054586668809254967
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,float16,15,0.06735999882221222
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,15,0.05392000079154968
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,float16,31,0.06760000189145406
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,31,0.05602133274078369
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,float16,63,0.06654400130112965
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,63,0.05492266515890757
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,float16,1,0.09160533547401428
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,1,0.08079466720422109
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,float16,3,0.09191999832789104
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,3,0.0804799993832906
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,float16,7,0.09146133065223694
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,7,0.08066666622956593
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,float16,15,0.0922933320204417
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,15,0.08092799782752991
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,float16,31,0.0926026701927185
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,31,0.08237866560618083
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,63,0.0809333324432373
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,float16,63,0.09249599774678548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,float16,1,0.12392533818880717
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,1,0.10213866829872131
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,float16,3,0.12141866485277812
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,3,0.1030560036500295
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,float16,7,0.12389866511027019
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,7,0.10214400291442871
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,float16,15,0.12141333023707072
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,15,0.10359467069307964
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,float16,31,0.12390933434168498
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,float16,1,0.1667520006497701
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,31,0.10167466600735982
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,1,0.15089600284894308
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,float16,3,0.1718239982922872
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,3,0.14838932951291403
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,float16,7,0.1718239982922872
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,7,0.15035200119018555
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,float16,15,0.17082667350769043
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,1,0.012261333564917246
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,1,0.010762666662534079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,3,0.01097600037852923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,3,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,7,0.01097600037852923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,7,0.010928000013033548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,15,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,15,0.01081066702802976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,31,0.010832000523805618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,31,0.01099733387430509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,63,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,15,0.14802133043607077
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,63,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,127,0.010911999891201654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,127,0.010965333630641302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,255,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,255,0.01081066702802976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,511,0.011242666592200598
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,1023,0.013173333058754602
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,511,0.013114667187134424
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,1023,0.01302933320403099
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,2047,0.014848000059525171
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,2047,0.014981333166360855
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,4095,0.029893333713213604
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,4095,0.01746133342385292
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,8191,0.048063998421033226
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,8191,0.03200000027815501
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,1,0.010821333775917688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,1,0.010784000158309937
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,3,0.010794666906197866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,3,0.010965333630641302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,7,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,7,0.010938666760921478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,float16,31,0.17185600598653158
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,15,0.011125333607196808
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,31,0.010992000500361124
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,15,0.011205332974592844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,31,0.010970667004585266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,63,0.011349332829316458
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,63,0.011045332998037338
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,127,0.010954666882753372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,127,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,255,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,255,0.011120000233252844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,511,0.01110400011142095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,511,0.011594666788975397
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,1023,0.013679999858140945
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,1023,0.013093333691358566
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,2047,0.029765332738558452
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,2047,0.015919999529918034
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,4095,0.046298667788505554
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,4095,0.030080000559488933
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,8191,0.04765866696834564
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,8191,0.08115733166535695
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,1,0.010816000401973724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,1,0.01097600037852923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,31,0.15067199865976968
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,3,0.01118933285276095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,3,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,7,0.011029332876205444
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,7,0.011237333218256632
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,15,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,31,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,15,0.010778666784365972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,31,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,63,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,63,0.010821333775917688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,127,0.011130666981140772
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,255,0.010944000134865442
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,127,0.011215999722480774
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,255,0.011264000087976456
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,511,0.014991999914248785
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,511,0.011221333096424738
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,1023,0.013082666943470636
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,2047,0.029552000264326733
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,1023,0.013232000172138214
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,2047,0.01709866647919019
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,4095,0.046485334634780884
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,4095,0.03152533372243246
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,1,0.011221333096424738
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,1,0.011909333368142446
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,3,0.010928000013033548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,3,0.011247999966144562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,7,0.011071999867757162
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,7,0.011477333803971609
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,15,0.011087999989589056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,15,0.011237333218256632
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,31,0.011077333241701126
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,31,0.011920000116030375
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,63,0.011125333607196808
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,63,0.01129066695769628
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,127,0.012671999633312225
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,127,0.013173333058754602
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,255,0.011296000331640244
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,255,0.012917333592971167
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,511,0.014826666563749313
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,1023,0.016885332763195038
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,1023,0.029440000653266907
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,511,0.013386666774749756
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,2047,0.0315733328461647
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,2047,0.04905599852403005
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,4095,0.08196266492207845
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,4095,0.04864533245563507
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,1,0.010682666053374609
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,3,0.010191999996701876
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,1,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,7,0.01081066702802976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,3,0.010682666053374609
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,7,0.010698666175206503
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,15,0.009824000298976898
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,15,0.010645333677530289
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,31,0.010234666367371878
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,31,0.010869332899649939
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,63,0.009866666669646898
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,63,0.011050666371981302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,127,0.009136000027259191
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,255,0.00960533320903778
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,127,0.010885333021481832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,255,0.010640000303586325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,511,0.011018666128317514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,511,0.010874666273593903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,1023,0.01118933285276095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,1023,0.011663999408483505
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,2047,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,2047,0.011125333607196808
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,4095,0.012810666114091873
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,4095,0.01232533281048139
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,8191,0.016997333616018295
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,8191,0.017093333105246227
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,16383,0.019285333653291065
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,16383,0.019258666783571243
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,1,0.010842667271693548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,1,0.00892800030608972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,3,0.011034666250149408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,3,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,7,0.009786666681369146
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,7,0.01071999967098236
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,15,0.00973866693675518
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,15,0.011002667248249054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,31,0.010784000158309937
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,31,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,63,0.008863999818762144
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,63,0.010656000425418219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,127,0.009279999881982803
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,127,0.009365333244204521
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,255,0.010661333799362183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,255,0.009183999771873156
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,511,0.01368533323208491
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,511,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,1023,0.011098666737476984
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,1023,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,2047,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,2047,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,4095,0.013023999830087027
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,4095,0.013248000293970108
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,8191,0.015082667271296183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,8191,0.015061333775520325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,16383,0.01746133342385292
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,1,0.009098666409651438
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,16383,0.01646399994691213
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,1,0.010794666906197866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,3,0.009173333023985228
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,3,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,7,0.010735999792814255
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,7,0.010746666540702185
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,15,0.010832000523805618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,15,0.010693332801262537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,31,0.009114666531483332
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,31,0.010874666273593903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,63,0.01071999967098236
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,127,0.009039999917149544
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,63,0.011130666981140772
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,127,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,255,0.01119999960064888
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,255,0.010954666882753372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,511,0.011018666128317514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,1023,0.011402666568756104
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,511,0.011941333611806234
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,1023,0.01097600037852923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,2047,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,2047,0.011221333096424738
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,4095,0.015103999525308609
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,8191,0.017082666357358296
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,4095,0.01613333324591319
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,8191,0.017338667064905167
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,16383,0.019093333433071773
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,16383,0.017136000096797943
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,1,0.011087999989589056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,3,0.010816000401973724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,3,0.010181333248813948
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,1,0.010928000013033548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,7,0.010512000570694605
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,7,0.00915733352303505
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,15,0.009242666885256767
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,15,0.01137599969903628
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,31,0.010698666175206503
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,31,0.009178666397929192
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,63,0.009850666547815004
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,63,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,127,0.009530666594703993
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,127,0.010672000547250112
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,255,0.009477333476146063
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,255,0.010928000013033548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,511,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,511,0.010735999792814255
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,1023,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,2047,0.012826666235923767
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,1023,0.010703999549150467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,2047,0.012879999975363413
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,4095,0.01312000056107839
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,4095,0.014309333016475042
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,8191,0.015253332753976187
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,8191,0.016261332978804905
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,16383,0.03136000037193298
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,16383,0.01730666682124138
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,1,0.012928000340859095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,1,0.013061333447694778
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,3,0.012960000584522883
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,3,0.012826666235923767
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,7,0.0129120002190272
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,7,0.013183999806642532
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,15,0.013157332936922709
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,15,0.012991999586423239
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,31,0.012853333105643591
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,31,0.012826666235923767
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,63,0.012949333836634954
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,63,0.012805332740147909
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,127,0.012885333349307379
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,255,0.012901333471139273
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,255,0.012944000462690989
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,127,0.012714666624863943
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,511,0.015024000157912573
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,511,0.015018666783968607
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,1023,0.02972800036271413
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,1023,0.017290666699409485
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,2047,0.047839999198913574
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,2047,0.03180266668399175
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,1,0.014309333016475042
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,1,0.013013333082199097
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,3,0.015040000279744467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,3,0.013034666577974955
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,7,0.014874666929244995
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,7,0.013034666577974955
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,15,0.013722666849692663
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,15,0.013237333546082178
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,31,0.013088000317414602
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,63,0.014298666268587112
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,31,0.01544533297419548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,63,0.013167999684810638
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,127,0.014837333311637243
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,127,0.01312000056107839
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,255,0.013744000345468521
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,255,0.014864000181357065
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,511,0.02959999938805898
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,511,0.01710933322707812
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,1023,0.04809600114822388
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,1,0.010565333068370819
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,1023,0.031871999303499855
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,1,0.01102399950226148
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,2047,0.08349333206812541
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,2047,0.05041599770387014
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,3,0.010741333166758219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,3,0.01081066702802976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,7,0.009242666885256767
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,7,0.010965333630641302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,15,0.010741333166758219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,31,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,15,0.010735999792814255
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,31,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,63,0.010650667051474253
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,63,0.010847999403874079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,127,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,127,0.010741333166758219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,255,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,255,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,511,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,511,0.01128000020980835
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,1023,0.011226666470368704
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,1023,0.010938666760921478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,2047,0.013002666334311167
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,4095,0.015194666882356008
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,2047,0.014314666390419006
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,4095,0.015002666662136713
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,8191,0.017008000363906223
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,8191,0.017114666601022083
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,16383,0.033770665526390076
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,16383,0.01952533299724261
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,1,0.008805333326260248
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,1,0.010757333288590113
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,3,0.010784000158309937
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,7,0.008943999807039896
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,3,0.01080000028014183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,7,0.010682666053374609
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,15,0.01022933361430963
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,15,0.00919999989370505
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,31,0.010821333775917688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,31,0.010741333166758219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,63,0.00914666677514712
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,63,0.008912000184257826
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,127,0.010608000059922537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,127,0.010885333021481832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,255,0.009999999776482582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,255,0.008912000184257826
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,511,0.011359999577204386
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,511,0.011120000233252844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,1023,0.011146667102972666
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,1023,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,2047,0.013125333935022354
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,2047,0.013050666699806849
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,4095,0.012757333616415659
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,4095,0.01301866645614306
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,8191,0.02945599953333537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,8191,0.015599999576807022
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,16383,0.03035733352104823
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,16383,0.04781333108743032
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,1,0.017018667111794155
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,1,0.015301333119471868
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,3,0.015034666905800501
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,3,0.015087999403476715
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,7,0.017008000363906223
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,15,0.01525866612792015
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,15,0.014970666418472925
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,7,0.015253332753976187
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,31,0.016943999876578648
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,31,0.015040000279744467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,63,0.015040000279744467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,63,0.01534933348496755
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,127,0.01709866647919019
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,127,0.015034666905800501
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,255,0.01532799998919169
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,255,0.015146666516860327
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,511,0.03153600047032038
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,511,0.01894933357834816
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,1023,0.0497920016447703
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,1,0.019152000546455383
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,1023,0.033589333295822144
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,1,0.017077332983414333
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,3,0.01931200052301089
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,7,0.019199999670187633
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,3,0.017077332983414333
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,15,0.01904533306757609
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,15,0.017008000363906223
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,31,0.018863999595244724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,7,0.01806933308641116
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,31,0.017221332838137943
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,63,0.019146667172511418
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,63,0.017322666943073273
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,127,0.0169813334941864
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,127,0.01940800001223882
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,255,0.02958400050799052
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,255,0.01739199956258138
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,511,0.04822933177153269
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,511,0.03126399964094162
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,1023,0.0831520011027654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,1023,0.05012266834576925
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,1,0.021498667697111767
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,3,0.01932266727089882
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,3,0.021290667355060577
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,1,0.019914666811625164
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,7,0.021370666722456615
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,7,0.019482667247454327
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,15,0.021290667355060577
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,15,0.019343999524911244
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,31,0.02141333371400833
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,31,0.019386666516462963
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,63,0.021386665602525074
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,63,0.019509332875410717
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,127,0.021925332645575207
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,127,0.01958400011062622
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,255,0.03141333411137263
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,255,0.019893333315849304
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,1,0.027322667340437572
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,1,0.02550933261712392
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,3,0.027701333165168762
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,3,0.025722667574882507
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,7,0.027248000105222065
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,7,0.025472000241279602
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,15,0.02757333219051361
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,15,0.025402667621771496
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,31,0.02736533433198929
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,31,0.02550933261712392
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,63,0.027482666075229645
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,63,0.025568000972270966
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,127,0.03287466615438461
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,127,0.025546667476495106
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,255,0.04816000163555145
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,255,0.03341866781314214
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,1,0.035391998787721
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,1,0.031514666974544525
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,3,0.035461333890755974
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,3,0.03142933299144109
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,7,0.03554133325815201
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,7,0.031258667508761086
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,15,0.03549333413441976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,15,0.03151999910672506
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,31,0.03583999971548716
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,31,0.03127466638882955
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,63,0.03545066714286804
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,63,0.031557333966096245
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,127,0.03892799963553747
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,127,0.03169599920511246
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,1,0.0503359983364741
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,1,0.04376000165939331
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,3,0.05040533343950907
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,3,0.04364266494909922
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,7,0.04995200037956238
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,7,0.04348266621430715
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,15,0.05055999755859375
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,15,0.04378666480382284
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,31,0.04987733562787374
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,31,0.043023998538653054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,63,0.05073600014050802
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,1,0.010778666784365972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,1,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,63,0.04387199878692627
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,3,0.009797333429257074
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,7,0.008890666688481966
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,3,0.011114666859308878
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,7,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,15,0.010816000401973724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,127,0.055200000603993736
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,15,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,31,0.011125333607196808
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,63,0.010970667004585266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,63,0.010847999403874079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,31,0.01091733326514562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,127,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,127,0.011141333729028702
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,255,0.010832000523805618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,255,0.010938666760921478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,511,0.011157333850860596
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,511,0.011045332998037338
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,1023,0.011567999919255575
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,1023,0.01240533341964086
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,2047,0.013045333325862885
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,2047,0.01312000056107839
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,4095,0.015040000279744467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,4095,0.015200000256299973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,8191,0.029802667597929638
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,127,0.04604266583919525
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,8191,0.01722666621208191
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,16383,0.04761599997679392
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,1,0.011050666371981302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,1,0.010138666878143946
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,16383,0.03145066648721695
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,3,0.010911999891201654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,3,0.009173333023985228
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,7,0.01091733326514562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,7,0.010586666564146677
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,15,0.00898133342464765
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,15,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,31,0.009258666386206945
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,31,0.01101333275437355
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,63,0.0103946669648091
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,63,0.010837333897749582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,127,0.01098666712641716
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,127,0.010965333630641302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,255,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,511,0.01098666712641716
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,255,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,511,0.010938666760921478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,1023,0.01312000056107839
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,1023,0.01292266696691513
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,2047,0.014250667144854864
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,2047,0.012970666090647379
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,4095,0.015311999867359797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,4095,0.029466666281223297
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,8191,0.03145600110292435
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,8191,0.04771733283996582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,16383,0.08085333307584126
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,16383,0.0491946687301
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,float16,1,0.06635733445485432
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,float16,3,0.06649066507816315
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,3,0.055018668373425804
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,1,0.05537599821885427
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,float16,7,0.06609066824118297
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,7,0.053786665201187134
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,float16,15,0.06653866668542226
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,15,0.0543039987484614
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,float16,31,0.06630933284759521
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,31,0.05421866476535797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,float16,63,0.06697600086530049
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,63,0.05414933462937673
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,1,0.08067733546098073
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,float16,1,0.09113066395123799
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,float16,3,0.09173867106437683
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,3,0.08108800152937572
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,float16,7,0.09062932928403218
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,7,0.08054933448632558
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,float16,15,0.09239466985066731
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,15,0.08131733536720276
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,float16,31,0.0902880032857259
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,31,0.08061866462230682
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,63,0.08121066788832347
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,float16,63,0.09146666526794434
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,1,0.10161599516868591
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,float16,1,0.12177600463231404
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,float16,3,0.12319999933242798
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,3,0.10244266192118327
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,float16,7,0.12398933370908101
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,7,0.10121066371599834
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,float16,15,0.1222826639811198
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,15,0.10188800096511841
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,float16,31,0.1242133378982544
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,31,0.10141866405804952
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,float16,1,0.17019200325012207
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,1,0.1492533286412557
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,float16,3,0.17128533124923706
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,3,0.14847999811172485
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,float16,7,0.16995733976364136
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,7,0.1497173309326172
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,float16,15,0.1706399917602539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,1,0.010981333752473196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,1,0.011061333119869232
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,3,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,3,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,7,0.010874666273593903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,7,0.01073066641887029
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,15,0.010842667271693548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,15,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,31,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,31,0.010965333630641302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,63,0.010693332801262537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,63,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,127,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,127,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,255,0.010992000500361124
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,255,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,511,0.011173332730929056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,15,0.14832533399264017
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,511,0.011061333119869232
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,1023,0.013130666067202887
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,1023,0.012917333592971167
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,2047,0.014965333044528961
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,2047,0.014965333044528961
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,float16,31,0.16985066731770834
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,4095,0.030373332401116688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,31,0.14946666359901428
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,4095,0.017349333812793095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,8191,0.03141866624355316
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,8191,0.048437332113583885
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,1,0.01091733326514562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,3,0.01128000020980835
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,1,0.010981333752473196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,3,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,7,0.010885333021481832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,7,0.010944000134865442
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,15,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,15,0.011120000233252844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,31,0.01080000028014183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,31,0.01099733387430509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,63,0.010970667004585266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,63,0.011215999722480774
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,127,0.01121066634853681
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,127,0.010944000134865442
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,255,0.010874666273593903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,255,0.011071999867757162
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,511,0.012304000556468964
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,511,0.011247999966144562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,1023,0.01303999995191892
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,1023,0.013061333447694778
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,2047,0.029472000896930695
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,2047,0.015157333264748255
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,4095,0.0462666650613149
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,4095,0.029578665892283123
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,8191,0.08080533146858215
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,8191,0.047872001926104225
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,1,0.010581333190202713
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,3,0.010709332923094431
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,1,0.010832000523805618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,3,0.010928000013033548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,7,0.011146667102972666
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,7,0.010703999549150467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,15,0.010911999891201654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,15,0.011077333241701126
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,31,0.010853332777818045
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,31,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,63,0.01119999960064888
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,63,0.010794666906197866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,127,0.011157333850860596
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,127,0.011066666493813196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,255,0.010725333044926325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,255,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,511,0.012608000387748083
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,511,0.012858666479587555
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,1023,0.012858666479587555
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,1023,0.012954667210578918
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,2047,0.015365333606799444
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,2047,0.015200000256299973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,4095,0.03068266560633977
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,4095,0.01714666684468587
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,8191,0.050069332122802734
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,1,0.010885333021481832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,8191,0.032672000428040825
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,1,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,3,0.010869332899649939
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,7,0.010842667271693548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,3,0.010762666662534079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,7,0.010746666540702185
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,15,0.01102399950226148
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,15,0.010832000523805618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,31,0.011194666226704916
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,31,0.011125333607196808
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,63,0.010938666760921478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,63,0.01097600037852923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,127,0.011061333119869232
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,127,0.011296000331640244
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,255,0.010725333044926325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,255,0.010842667271693548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,511,0.011007999380429586
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,511,0.011626667032639185
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,1023,0.012901333471139273
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,1023,0.012896000097195307
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,2047,0.029546665648619335
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,2047,0.015637333194414776
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,4095,0.029946667452653248
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,4095,0.04622933268547058
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,8191,0.08109333117802937
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,8191,0.04806933303674062
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,1,0.010911999891201654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,1,0.011055999745925268
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,3,0.01118933285276095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,3,0.011434666812419891
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,7,0.011039999624093374
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,7,0.011109333485364914
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,15,0.012469333906968435
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,15,0.011370666325092316
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,31,0.010885333021481832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,31,0.01121066634853681
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,63,0.012869333227475485
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,63,0.012725333372751871
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,127,0.011317333827416102
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,127,0.011215999722480774
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,255,0.012549333274364471
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,255,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,511,0.014864000181357065
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,511,0.013429333766301474
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,1023,0.029418667157491047
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,1023,0.015285332997639975
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,2047,0.04797866443792979
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,2047,0.0310506671667099
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,4095,0.08181866506735484
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,4095,0.04801600178082784
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,1,0.010837333897749582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,8191,0.15061333775520325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,8191,0.08313600222269694
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,1,0.010682666053374609
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,3,0.00897066667675972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,3,0.008992000172535578
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,7,0.010757333288590113
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,7,0.009173333023985228
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,15,0.00877333308259646
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,15,0.010821333775917688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,31,0.008853333070874214
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,31,0.00897066667675972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,63,0.010368000095089277
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,63,0.008890666688481966
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,127,0.008943999807039896
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,127,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,255,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,511,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,511,0.011407999942700068
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,255,0.011567999919255575
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,1023,0.010837333897749582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,1023,0.010885333021481832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,2047,0.011343999455372492
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,4095,0.011616000284751257
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,2047,0.012666666259368261
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,4095,0.011077333241701126
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,8191,0.01703466723362605
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,8191,0.017045332739750545
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,16383,0.021082667013009388
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,32767,0.02348800003528595
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,16383,0.021749332547187805
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,32767,0.026170666019121807
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,1,0.008885333314538002
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,1,0.010741333166758219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,3,0.010442666709423065
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,3,0.011450666934251785
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,7,0.008901333436369896
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,7,0.010832000523805618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,15,0.00916800027092298
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,31,0.009194666519761086
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,31,0.010661333799362183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,15,0.01202133297920227
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,63,0.009765333185593287
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,63,0.010506667196750641
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,127,0.010778666784365972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,255,0.00914666677514712
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,127,0.011071999867757162
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,255,0.00916800027092298
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,511,0.010821333775917688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,511,0.010661333799362183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,1023,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,1023,0.011296000331640244
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,2047,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,2047,0.010693332801262537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,4095,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,8191,0.014943999548753103
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,8191,0.015295999745527903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,4095,0.011407999942700068
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,16383,0.016986666868130367
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,16383,0.01706133286158244
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,32767,0.01929066702723503
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,32767,0.01951466624935468
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,1,0.009152000149091085
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,1,0.00884799969693025
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,3,0.010026666646202406
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,3,0.011157333850860596
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,7,0.009743999689817429
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,7,0.009125333279371262
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,15,0.010144000252087912
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,15,0.01128000020980835
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,31,0.009141333401203156
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,31,0.008837333569924036
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,63,0.01002133327225844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,127,0.00922133338948091
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,127,0.009050666665037474
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,63,0.011045332998037338
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,255,0.010725333044926325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,255,0.00891733355820179
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,511,0.010954666882753372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,511,0.011264000087976456
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,1023,0.010821333775917688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,1023,0.01091733326514562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,2047,0.010837333897749582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,2047,0.011440000186363855
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,4095,0.01312000056107839
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,4095,0.012954667210578918
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,8191,0.015439999600251516
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,8191,0.01552533358335495
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,16383,0.017136000096797943
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,16383,0.017301333447297413
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,1,0.010703999549150467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,32767,0.03339199970165888
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,32767,0.018357332795858383
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,3,0.008912000184257826
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,1,0.010970667004585266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,3,0.009045333291093508
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,7,0.009082666908701261
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,7,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,15,0.009248000259200731
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,15,0.009461333354314169
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,31,0.010762666662534079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,31,0.009183999771873156
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,63,0.010698666175206503
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,63,0.011146667102972666
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,127,0.00890666681031386
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,127,0.011120000233252844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,255,0.010698666175206503
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,255,0.010816000401973724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,511,0.010938666760921478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,511,0.011402666568756104
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,1023,0.010954666882753372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,1023,0.010911999891201654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,2047,0.01126933346192042
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,2047,0.010874666273593903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,4095,0.011178666104873022
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,4095,0.015365333606799444
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,8191,0.01720000058412552
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,8191,0.01708799973130226
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,16383,0.019365333020687103
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,16383,0.018853332847356796
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,32767,0.021189334491888683
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,32767,0.020970667401949566
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,1,0.009152000149091085
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,1,0.010703999549150467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,3,0.010618666807810465
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,3,0.010954666882753372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,7,0.00915733352303505
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,7,0.011445333560307821
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,15,0.010640000303586325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,15,0.010656000425418219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,31,0.009098666409651438
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,63,0.01101333275437355
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,63,0.00890666681031386
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,31,0.01110400011142095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,127,0.008896000062425932
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,127,0.010512000570694605
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,255,0.008869333192706108
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,511,0.010944000134865442
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,255,0.011429333438475927
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,511,0.010672000547250112
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,1023,0.010778666784365972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,1023,0.011061333119869232
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,2047,0.011071999867757162
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,2047,0.01128000020980835
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,4095,0.012863999853531519
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,4095,0.013023999830087027
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,8191,0.015226667126019796
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,8191,0.015274666249752045
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,16383,0.015989333391189575
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,16383,0.01727466657757759
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,32767,0.03229333211978277
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,32767,0.019530666371186573
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,1,0.008810666700204214
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,1,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,3,0.009824000298976898
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,3,0.010703999549150467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,7,0.010944000134865442
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,7,0.009583999713261923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,15,0.00927466650803884
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,15,0.0107893335322539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,31,0.010821333775917688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,31,0.008901333436369896
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,63,0.01051733394463857
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,127,0.009103999783595404
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,127,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,63,0.010741333166758219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,255,0.010565333068370819
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,255,0.009098666409651438
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,511,0.0107893335322539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,511,0.011178666104873022
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,1023,0.010911999891201654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,1023,0.01099733387430509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,2047,0.012863999853531519
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,2047,0.012629333883523941
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,4095,0.014853333433469137
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,4095,0.01301866645614306
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,8191,0.015589332828919092
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,8191,0.01533866673707962
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,16383,0.01701333373785019
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,16383,0.03169066707293192
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,32767,0.0495306650797526
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,32767,0.03376533339420954
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,1,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,1,0.011338666081428528
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,3,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,3,0.011264000087976456
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,7,0.01080000028014183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,7,0.011242666592200598
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,15,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,15,0.011391999820868174
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,31,0.010837333897749582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,31,0.010911999891201654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,63,0.011055999745925268
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,63,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,127,0.010837333897749582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,255,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,127,0.011071999867757162
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,255,0.010746666540702185
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,511,0.012639999389648438
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,511,0.012938667088747025
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,1023,0.015130666395028433
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,1023,0.012965332716703415
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,2047,0.03009066730737686
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,2047,0.017050666113694508
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,4095,0.0310506671667099
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,4095,0.04857600231965383
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,1,0.011045332998037338
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,1,0.012981332838535309
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,3,0.011205332974592844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,7,0.01268799975514412
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,3,0.01166933278242747
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,7,0.011194666226704916
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,15,0.011029332876205444
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,15,0.010885333021481832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,31,0.011226666470368704
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,63,0.011194666226704916
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,31,0.012015999605258306
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,63,0.011066666493813196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,127,0.011519999553759893
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,127,0.013162666310866674
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,255,0.012272000312805176
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,255,0.011077333241701126
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,511,0.014906667172908783
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,511,0.013248000293970108
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,1023,0.02957333376010259
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,1023,0.014991999914248785
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,2047,0.04781866570313772
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,2047,0.03170666595300039
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,1,0.01516266663869222
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,4095,0.049546668926874794
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,4095,0.08311466872692108
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,1,0.014074667046467463
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,3,0.012858666479587555
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,3,0.014666666587193808
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,7,0.015103999525308609
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,7,0.01381333296497663
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,15,0.012938667088747025
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,15,0.014943999548753103
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,31,0.014906667172908783
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,31,0.014501333236694336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,63,0.013023999830087027
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,63,0.014864000181357065
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,127,0.014906667172908783
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,127,0.013338666409254074
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,255,0.012981332838535309
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,255,0.014906667172908783
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,511,0.02962133288383484
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,1023,0.031717332700888314
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,1023,0.04765866696834564
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,511,0.01722666621208191
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,2047,0.08393067121505737
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,2047,0.049685334165891014
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,1,0.010645333677530289
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,4095,0.15380799770355225
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,3,0.010682666053374609
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,1,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,4095,0.0851146678129832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,3,0.009066666786869368
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,7,0.010805333654085795
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,7,0.011029332876205444
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,15,0.00916800027092298
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,31,0.010751999914646149
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,31,0.010735999792814255
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,15,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,63,0.00895999992887179
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,127,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,63,0.010666667173306147
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,127,0.01129066695769628
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,255,0.010928000013033548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,255,0.008863999818762144
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,511,0.011152000476916632
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,511,0.011551999797423681
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,1023,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,1023,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,2047,0.011247999966144562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,2047,0.01139733319481214
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,4095,0.01458666721979777
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,4095,0.014848000059525171
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,8191,0.01714666684468587
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,8191,0.01708799973130226
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,16383,0.017984000345071156
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,16383,0.017370666066805523
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,32767,0.03338133295377096
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,32767,0.021253332495689392
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,1,0.00903466654320558
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,1,0.010885333021481832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,3,0.010816000401973724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,3,0.010703999549150467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,7,0.010597333312034607
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,7,0.009178666397929192
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,15,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,15,0.010938666760921478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,31,0.008832000195980072
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,31,0.011077333241701126
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,63,0.009103999783595404
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,63,0.010474666953086853
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,127,0.010794666906197866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,127,0.008890666688481966
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,255,0.009626666704813639
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,255,0.010709332923094431
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,511,0.01091733326514562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,511,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,1023,0.010847999403874079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,1023,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,2047,0.012986666212479273
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,2047,0.01463466634353002
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,4095,0.013295999417702356
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,4095,0.014463999619086584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,8191,0.01681600014368693
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,8191,0.01505600040157636
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,16383,0.03133866687615713
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,16383,0.017488000293572743
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,32767,0.04809600114822388
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,32767,0.032111999889214836
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,1,0.010304000228643417
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,1,0.009258666386206945
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,3,0.008832000195980072
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,3,0.010709332923094431
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,7,0.010474666953086853
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,7,0.009050666665037474
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,15,0.0100853331387043
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,15,0.010885333021481832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,31,0.009114666531483332
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,63,0.010480000327030817
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,63,0.010661333799362183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,31,0.011391999820868174
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,127,0.010586666564146677
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,127,0.010645333677530289
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,255,0.010250666489203772
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,511,0.010762666662534079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,255,0.011648000528415045
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,511,0.01108266661564509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,1023,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,1023,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,2047,0.012730666746695837
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,4095,0.013722666849692663
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,2047,0.013232000172138214
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,4095,0.013141332815090815
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,8191,0.02993600070476532
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,8191,0.01522133375207583
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,16383,0.031008000175158184
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,16383,0.04708800216515859
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,32767,0.08078933258851369
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,32767,0.04753600060939789
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,1,0.012991999586423239
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,3,0.013141332815090815
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,3,0.012975999464591345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,1,0.013002666334311167
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,7,0.012991999586423239
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,15,0.013002666334311167
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,7,0.012789333860079447
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,31,0.012986666212479273
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,15,0.013114667187134424
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,31,0.013104000439246496
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,63,0.0129120002190272
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,63,0.012981332838535309
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,127,0.012928000340859095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,127,0.013199999928474426
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,255,0.013189333180586496
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,255,0.012981332838535309
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,511,0.014975999792416891
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,1023,0.03148799886306127
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,511,0.015205333630243937
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,1023,0.017103999853134155
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,2047,0.04882133503754934
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,1,0.01591466615597407
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,2047,0.03297599901755651
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,1,0.012991999586423239
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,3,0.014997333288192749
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,3,0.015114666273196539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,7,0.013157332936922709
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,7,0.014954666296641031
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,15,0.014864000181357065
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,31,0.014991999914248785
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,31,0.014959999670584997
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,15,0.015130666395028433
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,63,0.013162666310866674
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,63,0.014901333798964819
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,127,0.01676799977819125
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,127,0.015114666273196539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,255,0.015087999403476715
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,255,0.013194666554530462
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,511,0.030591999491055805
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,511,0.017594666530688603
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,1023,0.03166933357715607
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,1023,0.04788800080617269
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,2047,0.0846453309059143
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,2047,0.05114666620890299
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,3,0.019002666076024372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,1,0.01729600007335345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,1,0.018885333091020584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,3,0.01738133281469345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,7,0.017029333859682083
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,7,0.019018666197856266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,15,0.01700266698996226
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,15,0.018917333334684372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,31,0.01930133377512296
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,63,0.019141333798567455
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,63,0.01757866640885671
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,127,0.018885333091020584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,31,0.01770666614174843
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,127,0.017050666113694508
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,255,0.030832000076770782
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,255,0.016986666868130367
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,511,0.03172266731659571
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,1023,0.0827893316745758
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,1023,0.04823466638724009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,511,0.04794133206208547
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,2047,0.0842026670773824
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,2047,0.15036267042160034
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,1,0.015333333363135656
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,1,0.015130666395028433
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,3,0.016986666868130367
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,3,0.014959999670584997
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,7,0.01504533365368843
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,7,0.015461333096027374
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,15,0.01516266663869222
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,15,0.015034666905800501
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,31,0.015354666858911514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,31,0.015402667224407196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,63,0.01551466683546702
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,63,0.014975999792416891
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,127,0.01573866605758667
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,127,0.01515199989080429
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,255,0.015615999698638916
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,255,0.014959999670584997
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,511,0.03143999973932902
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,511,0.019567999988794327
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,1,0.01921066641807556
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,1,0.01722666621208191
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,3,0.019061333189407986
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,3,0.01735466718673706
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,7,0.01889066646496455
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,7,0.01735466718673706
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,15,0.018960000326236088
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,31,0.01907733331123988
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,15,0.017674667139848072
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,31,0.016976000120242436
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,63,0.019141333798567455
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,63,0.017621333400408428
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,127,0.017055999487638474
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,127,0.01886933296918869
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,255,0.029866665601730347
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,255,0.017637333522240322
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,511,0.04788800080617269
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,511,0.03162666658560435
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,1,0.027978666126728058
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,1,0.025402667621771496
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,3,0.027349332968393963
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,3,0.02516266703605652
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,7,0.027327999472618103
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,7,0.025536000728607178
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,15,0.027232001225153606
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,15,0.02536533276240031
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,31,0.02775466690460841
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,31,0.025472000241279602
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,63,0.02722666660944621
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,63,0.025242666403452556
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,127,0.033376000821590424
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,127,0.025850666066010792
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,255,0.04781333108743032
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,255,0.032511999209721885
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,511,0.08285333216190338
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,511,0.050101334849993386
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,1,0.02146666745344798
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,1,0.019253333409627277
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,3,0.021536000072956085
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,3,0.01969066634774208
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,7,0.021136000752449036
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,7,0.019354666272799175
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,15,0.02184533327817917
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,15,0.019381333142518997
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,31,0.021840001145998638
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,31,0.019509332875410717
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,63,0.02146666745344798
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,63,0.01941866676012675
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,127,0.02254933367172877
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,255,0.03169599920511246
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,127,0.019765333582957584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,255,0.019685332973798115
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,1,0.029322666426499683
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,1,0.027285332481066387
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,3,0.029802667597929638
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,3,0.027445333699385326
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,7,0.02934933453798294
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,7,0.02739733209212621
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,15,0.02958400050799052
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,15,0.027477333943049114
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,31,0.02924266705910365
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,31,0.027280000348885853
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,63,0.029898665845394135
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,63,0.027237333357334137
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,127,0.035605333745479584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,127,0.027482666075229645
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,255,0.03541333228349686
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,255,0.051594664653142296
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,1,0.0498879998922348
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,1,0.04246933261553446
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,3,0.05023466547330221
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,3,0.04200000067551931
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,7,0.049925332268079124
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,7,0.04234133164087931
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,15,0.049786667029062905
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,15,0.041877334316571556
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,31,0.05077866713205973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,31,0.042277331153551735
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,63,0.05046933392683665
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,63,0.04204800228277842
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,127,0.05450133482615153
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,1,0.010762666662534079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,127,0.04629333317279816
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,1,0.0107893335322539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,3,0.010965333630641302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,3,0.010682666053374609
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,7,0.010970667004585266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,255,0.08687999844551086
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,7,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,15,0.01071999967098236
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,31,0.010693332801262537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,31,0.010992000500361124
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,63,0.01080000028014183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,15,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,63,0.011002667248249054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,127,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,127,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,255,0.01032533310353756
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,255,0.010645333677530289
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,511,0.01098666712641716
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,511,0.011328000575304031
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,255,0.055685331424077354
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,1023,0.01097600037852923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,1023,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,2047,0.013178666432698568
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,4095,0.01516266663869222
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,4095,0.015034666905800501
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,8191,0.016890666137139004
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,8191,0.01724799970785777
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,16383,0.02125866711139679
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,16383,0.03323200096686681
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,2047,0.013248000293970108
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,32767,0.05156800150871277
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,32767,0.03626666714747747
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,1,0.009658666948477427
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,1,0.01081066702802976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,3,0.010543999572594961
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,3,0.010938666760921478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,7,0.009653333574533463
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,7,0.010784000158309937
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,15,0.010816000401973724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,15,0.010954666882753372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,31,0.008890666688481966
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,31,0.010757333288590113
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,63,0.01028266673286756
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,63,0.010634666929642359
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,127,0.009056000038981438
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,127,0.010645333677530289
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,255,0.00902399979531765
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,255,0.011087999989589056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,511,0.011018666128317514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,511,0.011120000233252844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,1023,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,1023,0.011183999478816986
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,2047,0.012719999998807907
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,2047,0.012879999975363413
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,4095,0.015018666783968607
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,4095,0.013183999806642532
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,8191,0.029301332930723827
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,8191,0.017024000485738117
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,16383,0.04851733148097992
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,16383,0.031632001201311745
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,32767,0.08172800143559773
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,1,0.01110400011142095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,32767,0.04906666775544485
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,1,0.00980266680320104
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,7,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,3,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,3,0.01098666712641716
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,7,0.010725333044926325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,15,0.0107893335322539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,15,0.009519999846816063
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,31,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,31,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,63,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,63,0.01097600037852923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,127,0.008890666688481966
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,127,0.011205332974592844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,255,0.01098666712641716
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,255,0.010634666929642359
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,511,0.011050666371981302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,511,0.011312000453472137
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,1023,0.012890666723251343
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,1023,0.011114666859308878
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,2047,0.015034666905800501
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,2047,0.01301866645614306
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,4095,0.02940266579389572
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,4095,0.017173333714405697
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,8191,0.04743466774622599
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,8191,0.03030933439731598
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,16383,0.08211733400821686
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,16383,0.04828266799449921
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,32767,0.15001599987347922
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,32767,0.08347200353940327
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,1,0.03545066714286804
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,1,0.030346666773160298
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,3,0.035786665976047516
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,3,0.03125333289305369
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,7,0.036858665446440377
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,7,0.0313226655125618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,15,0.039408000806967415
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,15,0.03173333406448364
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,31,0.03619199991226196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,63,0.036746665835380554
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,63,0.03158933420976003
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,31,0.03151999910672506
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,127,0.038693333665529885
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,127,0.031744000812371574
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,1,0.054245332876841225
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,1,0.045754666129748024
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,3,0.05388799806435903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,3,0.04610666632652283
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,7,0.05418133238951365
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,7,0.04574933151404063
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,15,0.05421866476535797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,31,0.05449600021044413
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,15,0.04632000128428141
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,31,0.04585599899291992
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,63,0.05423999826113383
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,63,0.047541335225105286
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,127,0.06043733159701029
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,1,0.08975467085838318
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,127,0.05216533442338308
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,1,0.080485333998998
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,3,0.08083199958006541
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,3,0.0897653301556905
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,7,0.08922132849693298
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,7,0.08011733492215474
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,15,0.09108799695968628
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,31,0.09045333663622539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,15,0.07996266583601634
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,31,0.08062933385372162
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,63,0.09059733152389526
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,63,0.07886933286984761
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,127,0.1014453371365865
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,127,0.08258666594823201
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,float16,1,0.06474133332570393
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,1,0.05386666456858317
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,float16,3,0.06524799764156342
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,3,0.05287999908129374
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,float16,7,0.06443733473618825
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,7,0.05318933228651682
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,float16,15,0.06541333099206288
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,15,0.05393599967161814
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,float16,31,0.06479999919732411
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,31,0.052485331892967224
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,float16,63,0.06478400031725566
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,63,0.054192001620928444
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,float16,1,0.09763200084368388
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,1,0.0888213316599528
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,3,0.09056533376375835
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,float16,3,0.09959999720255534
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,float16,7,0.09859733780225118
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,7,0.08906666437784831
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,float16,15,0.09819733103116353
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,15,0.0912000040213267
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,float16,31,0.09716799855232239
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,31,0.08900266885757446
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,float16,63,0.0995199978351593
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,float16,1,0.16981865962346396
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,1,0.14551466703414917
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,63,0.08980266253153484
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,float16,3,0.16739734013875326
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,float16,7,0.16617066661516824
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,3,0.14799466729164124
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,7,0.1474613348642985
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,15,0.14501866698265076
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,float16,15,0.16817599534988403
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,float16,31,0.16941332817077637
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,1,0.010981333752473196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,1,0.010746666540702185
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,3,0.010970667004585266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,3,0.011338666081428528
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,7,0.011002667248249054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,7,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,15,0.01020800011853377
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,15,0.010911999891201654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,31,0.010197333370645842
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,31,0.14854933818181357
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,31,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,63,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,63,0.010496000448862711
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,127,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,127,0.010672000547250112
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,255,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,255,0.011002667248249054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,511,0.01102399950226148
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,511,0.011194666226704916
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,1023,0.011509332805871964
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,1023,0.011973333855470022
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,2047,0.012949333836634954
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,2047,0.013013333082199097
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,4095,0.01469333345691363
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,4095,0.01578666642308235
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,8191,0.030069333811601002
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,16383,0.048170665899912514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,8191,0.018122666825850803
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,16383,0.032085334261258446
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,1,0.011050666371981302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,1,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,3,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,3,0.011317333827416102
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,7,0.01098666712641716
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,63,0.14775466918945312
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,7,0.011296000331640244
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,15,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,31,0.010751999914646149
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,15,0.011039999624093374
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,63,0.010538666198650995
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,31,0.010805333654085795
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,63,0.011109333485364914
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,127,0.010885333021481832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,127,0.010847999403874079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,255,0.010709332923094431
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,255,0.011194666226704916
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,511,0.01138666644692421
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,511,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,1023,0.013056000073750814
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,1023,0.013343999783198038
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,2047,0.013007999708255133
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,2047,0.013242666920026144
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,4095,0.015423999478419622
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,4095,0.029904000461101532
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,8191,0.047050664822260536
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,8191,0.031221332649389904
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,16383,0.08229333162307739
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,16383,0.04937066634496053
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,1,0.010911999891201654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,1,0.010687999427318573
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,3,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,3,0.011050666371981302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,7,0.011152000476916632
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,7,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,15,0.010965333630641302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,15,0.011120000233252844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,31,0.011002667248249054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,31,0.010970667004585266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,63,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,63,0.010842667271693548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,127,0.011002667248249054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,127,0.010693332801262537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,255,0.010837333897749582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,255,0.010634666929642359
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,511,0.011183999478816986
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,511,0.010981333752473196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,float16,63,0.16940265893936157
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,1023,0.013365333278973898
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,1023,0.013056000073750814
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,2047,0.02886933336655299
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,2047,0.015317333241303762
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,4095,0.046341334780057274
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,4095,0.029359998802344005
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,8191,0.0481279989083608
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,16383,0.08124266564846039
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,16383,0.1498293379942576
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,8191,0.08085333307584126
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,1,0.01099733387430509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,1,0.011247999966144562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,3,0.010703999549150467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,3,0.01081066702802976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,7,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,7,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,15,0.010703999549150467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,15,0.010832000523805618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,31,0.010672000547250112
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,31,0.010944000134865442
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,63,0.010954666882753372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,63,0.011002667248249054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,127,0.011007999380429586
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,127,0.010645333677530289
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,255,0.011077333241701126
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,255,0.011045332998037338
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,511,0.011605333536863327
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,511,0.012639999389648438
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,1023,0.013253333667914072
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,1023,0.01303999995191892
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,2047,0.015061333775520325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,4095,0.031397332747777305
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,4095,0.01714666684468587
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,2047,0.01505600040157636
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,8191,0.050106664498647056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,8191,0.031701333820819855
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,1,0.010853332777818045
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,1,0.01098666712641716
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,3,0.010693332801262537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,3,0.011050666371981302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,7,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,7,0.010597333312034607
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,15,0.010965333630641302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,15,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,31,0.011125333607196808
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,31,0.010944000134865442
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,63,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,63,0.010656000425418219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,127,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,127,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,255,0.010768000036478043
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,255,0.0107893335322539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,511,0.011018666128317514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,511,0.011077333241701126
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,1023,0.012901333471139273
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,2047,0.02934933453798294
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,2047,0.015311999867359797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,4095,0.04637333254019419
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,1023,0.013749333719412485
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,4095,0.030106666187445324
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,8191,0.08086933195590973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,8191,0.04784533381462097
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,1,0.011215999722480774
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,3,0.012949333836634954
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,1,0.012063999970753988
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,3,0.01099733387430509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,7,0.010954666882753372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,7,0.011909333368142446
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,15,0.010885333021481832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,31,0.011183999478816986
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,31,0.012191999703645706
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,15,0.011930666863918304
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,63,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,63,0.011589333415031433
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,127,0.01303999995191892
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,127,0.011829332758982977
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,255,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,255,0.011055999745925268
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,511,0.014842666685581207
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,511,0.013461332768201828
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,1023,0.015018666783968607
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,1023,0.02924266705910365
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,2047,0.049135997891426086
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,4095,0.048010667165120445
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,4095,0.0820853312810262
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,2047,0.03136533250411352
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,1,0.009125333279371262
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,1,0.0107893335322539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,8191,0.15132266283035278
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,3,0.00903466654320558
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,8191,0.08306666711966197
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,7,0.011055999745925268
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,3,0.01126933346192042
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,7,0.008896000062425932
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,15,0.008821333448092142
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,31,0.008816000074148178
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,15,0.010709332923094431
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,31,0.011264000087976456
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,63,0.009477333476146063
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,63,0.009072000160813332
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,127,0.010778666784365972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,127,0.011029332876205444
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,255,0.009472000102202097
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,255,0.008986666798591614
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,511,0.010757333288590113
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,511,0.01145600030819575
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,1023,0.010911999891201654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,1023,0.010821333775917688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,2047,0.011173332730929056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,4095,0.011194666226704916
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,2047,0.011663999408483505
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,4095,0.011007999380429586
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,8191,0.017045332739750545
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,8191,0.015728000551462173
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,16383,0.020202666521072388
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,16383,0.021290667355060577
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,32767,0.0234400009115537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,32767,0.023269332945346832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,1,0.008837333569924036
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,1,0.010837333897749582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,3,0.010565333068370819
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,3,0.009103999783595404
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,7,0.009061333412925402
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,7,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,15,0.009232000137368837
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,15,0.009994666402538618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,31,0.008965333302815756
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,31,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,63,0.008853333070874214
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,63,0.010426666587591171
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,127,0.009152000149091085
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,255,0.008842666943868002
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,255,0.010565333068370819
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,127,0.010751999914646149
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,511,0.010847999403874079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,511,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,1023,0.010768000036478043
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,1023,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,2047,0.01080000028014183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,2047,0.011146667102972666
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,4095,0.011274666835864386
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,4095,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,8191,0.015290666371583939
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,8191,0.014831999937693277
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,16383,0.017024000485738117
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,16383,0.01740266631046931
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,32767,0.019189332922299702
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,32767,0.019226666539907455
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,1,0.010304000228643417
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,1,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,3,0.009125333279371262
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,3,0.011109333485364914
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,7,0.009888000165422758
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,7,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,15,0.010778666784365972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,15,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,31,0.009072000160813332
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,31,0.010778666784365972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,63,0.008896000062425932
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,63,0.010842667271693548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,127,0.010618666807810465
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,127,0.009093333035707474
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,255,0.00949866697192192
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,255,0.010768000036478043
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,511,0.01091733326514562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,1023,0.010965333630641302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,511,0.012240000069141388
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,1023,0.011077333241701126
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,2047,0.010944000134865442
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,2047,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,4095,0.013290667285521826
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,4095,0.013855999956528345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,8191,0.015168000012636185
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,8191,0.014858666807413101
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,16383,0.017429333180189133
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,32767,0.017077332983414333
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,1,0.010634666929642359
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,32767,0.03355200091997782
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,16383,0.01783466711640358
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,1,0.008842666943868002
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,3,0.010565333068370819
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,3,0.01091733326514562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,7,0.011237333218256632
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,7,0.009082666908701261
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,15,0.010693332801262537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,15,0.01116266722480456
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,31,0.009125333279371262
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,31,0.011301333705584208
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,63,0.010538666198650995
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,63,0.010794666906197866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,127,0.008965333302815756
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,255,0.010597333312034607
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,127,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,255,0.010874666273593903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,511,0.010847999403874079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,511,0.011194666226704916
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,1023,0.01116266722480456
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,1023,0.01137599969903628
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,2047,0.010869332899649939
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,4095,0.01128000020980835
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,2047,0.01126933346192042
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,4095,0.012917333592971167
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,8191,0.017157333592573803
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,8191,0.016437333077192307
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,16383,0.019253333409627277
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,16383,0.019333332777023315
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,32767,0.021082667013009388
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,32767,0.01939733326435089
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,1,0.009557333464423815
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,1,0.010874666273593903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,3,0.009183999771873156
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,3,0.009237333511312803
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,7,0.009898666913310686
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,7,0.01081066702802976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,15,0.008938666433095932
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,15,0.01080000028014183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,31,0.009322666873534521
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,31,0.010703999549150467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,63,0.009152000149091085
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,63,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,127,0.010698666175206503
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,127,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,255,0.00884799969693025
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,255,0.009824000298976898
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,511,0.01062400018175443
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,511,0.010944000134865442
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,1023,0.010794666906197866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,1023,0.011333333949247995
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,2047,0.011055999745925268
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,2047,0.010911999891201654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,4095,0.013082666943470636
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,4095,0.013669333110253016
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,8191,0.014815999815861383
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,8191,0.015274666249752045
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,16383,0.016938666502634685
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,16383,0.017664000391960144
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,32767,0.03333866596221924
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,32767,0.017344000438849132
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,1,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,1,0.010346666599313417
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,3,0.008885333314538002
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,3,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,7,0.010885333021481832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,7,0.010373333469033241
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,15,0.00955200009047985
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,15,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,31,0.010101333260536194
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,31,0.010938666760921478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,63,0.00973866693675518
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,63,0.011114666859308878
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,127,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,127,0.010853332777818045
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,255,0.010869332899649939
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,511,0.010965333630641302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,255,0.010602666685978571
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,511,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,1023,0.010725333044926325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,1023,0.011173332730929056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,2047,0.012975999464591345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,2047,0.012949333836634954
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,4095,0.015344000111023584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,4095,0.013242666920026144
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,8191,0.015279999623696009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,8191,0.015562667200962702
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,16383,0.03124266614516576
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,32767,0.04885333279768626
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,16383,0.019173332800467808
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,32767,0.033930666744709015
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,1,0.0107893335322539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,1,0.01098666712641716
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,3,0.011146667102972666
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,3,0.011098666737476984
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,7,0.011018666128317514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,7,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,15,0.011194666226704916
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,31,0.01099733387430509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,31,0.010944000134865442
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,15,0.011413333316644033
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,63,0.011285333583752314
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,63,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,127,0.011066666493813196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,127,0.011610666910807291
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,255,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,255,0.011274666835864386
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,511,0.012234666695197424
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,511,0.012080000092585882
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,1023,0.01321600005030632
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,1023,0.013450667262077332
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,2047,0.029711998999118805
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,4095,0.04794133206208547
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,2047,0.017077332983414333
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,4095,0.03134933362404505
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,1,0.011920000116030375
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,1,0.011509332805871964
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,3,0.011312000453472137
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,3,0.012655999511480331
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,7,0.012565333396196365
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,7,0.011125333607196808
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,15,0.013013333082199097
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,31,0.011839999506870905
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,15,0.011242666592200598
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,31,0.011125333607196808
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,63,0.010954666882753372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,63,0.012970666090647379
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,127,0.012960000584522883
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,127,0.011231999844312668
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,255,0.0116799995303154
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,255,0.012863999853531519
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,511,0.012997332960367203
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,511,0.015082667271296183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,1023,0.029333333174387615
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,2047,0.04781333108743032
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,2047,0.03169599920511246
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,1023,0.01685333376129468
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,4095,0.04951466619968414
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,4095,0.08225599924723308
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,1,0.015333333363135656
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,1,0.015210667004187902
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,3,0.014991999914248785
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,3,0.012842666357755661
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,7,0.014965333044528961
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,7,0.014538666854302088
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,15,0.014933332800865173
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,15,0.01302933320403099
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,31,0.01482133318980535
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,31,0.014426667243242264
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,63,0.01292266696691513
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,63,0.014890667051076889
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,127,0.014922666052977243
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,127,0.01504533365368843
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,255,0.013440000514189402
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,255,0.012954667210578918
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,511,0.02991466720898946
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,511,0.01704000060757001
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,1023,0.04760533571243286
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,1023,0.031231999397277832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,2047,0.08422399560610454
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,2047,0.050101334849993386
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,4095,0.15432533621788025
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,1,0.00902399979531765
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,1,0.010816000401973724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,3,0.009290666629870733
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,4095,0.08505599697430928
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,3,0.010981333752473196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,7,0.009237333511312803
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,7,0.010661333799362183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,15,0.009925333162148794
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,15,0.010944000134865442
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,31,0.010762666662534079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,31,0.01102399950226148
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,63,0.010853332777818045
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,63,0.010778666784365972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,127,0.009578666960199675
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,127,0.01009599988659223
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,255,0.010751999914646149
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,255,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,511,0.010938666760921478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,511,0.011194666226704916
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,1023,0.010885333021481832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,1023,0.011087999989589056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,2047,0.011343999455372492
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,2047,0.011183999478816986
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,4095,0.01482133318980535
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,4095,0.015040000279744467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,8191,0.017055999487638474
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,8191,0.01658133293191592
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,16383,0.01913600042462349
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,16383,0.017717332889636356
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,32767,0.034815999368826546
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,32767,0.021157334248224895
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,1,0.010911999891201654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,1,0.010645333677530289
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,3,0.01002133327225844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,3,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,7,0.00915733352303505
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,7,0.010762666662534079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,15,0.009423999736706415
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,31,0.008901333436369896
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,15,0.010981333752473196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,31,0.010559999694426855
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,63,0.01097600037852923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,63,0.008901333436369896
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,127,0.01073066641887029
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,127,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,255,0.009423999736706415
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,255,0.008912000184257826
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,511,0.011120000233252844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,511,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,1023,0.010821333775917688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,1023,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,2047,0.012831999609867731
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,2047,0.013034666577974955
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,4095,0.012975999464591345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,4095,0.015018666783968607
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,8191,0.016895999511082966
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,8191,0.015568000574906668
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,16383,0.031311998764673867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,16383,0.016965333372354507
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,32767,0.050000001986821495
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,32767,0.03403199960788091
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,1,0.010128000130256018
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,1,0.009093333035707474
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,3,0.010837333897749582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,3,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,7,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,7,0.009455999980370203
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,15,0.010570666442314783
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,15,0.010837333897749582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,31,0.009343999748428663
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,31,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,63,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,127,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,63,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,127,0.009136000027259191
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,255,0.01099733387430509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,255,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,511,0.010874666273593903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,511,0.01138666644692421
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,1023,0.01081066702802976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,1023,0.011173332730929056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,2047,0.012634667257467905
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,2047,0.013002666334311167
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,4095,0.013183999806642532
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,4095,0.012981332838535309
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,8191,0.029391999046007793
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,8191,0.01874133323629697
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,16383,0.03134933362404505
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,16383,0.0473280002673467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,32767,0.048528000712394714
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,32767,0.0806933343410492
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,1,0.012826666235923767
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,1,0.013162666310866674
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,3,0.013130666067202887
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,3,0.013050666699806849
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,7,0.012970666090647379
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,7,0.012901333471139273
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,15,0.012975999464591345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,15,0.012975999464591345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,31,0.012981332838535309
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,31,0.012960000584522883
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,63,0.013061333447694778
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,63,0.01293333371480306
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,127,0.013045333325862885
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,127,0.012906666845083237
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,255,0.013114667187134424
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,511,0.014906667172908783
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,511,0.015050667027632395
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,255,0.01333333303531011
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,1023,0.031290667752424874
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,1023,0.01714133347074191
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,2047,0.04841066896915436
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,1,0.01482133318980535
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,1,0.01310933381319046
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,2047,0.03277866790692011
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,3,0.013295999417702356
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,7,0.014335999886194864
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,3,0.015194666882356008
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,15,0.01302933320403099
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,15,0.01504533365368843
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,31,0.014896000425020853
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,7,0.013077333569526672
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,31,0.012949333836634954
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,63,0.013669333110253016
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,63,0.014912000546852747
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,127,0.014874666929244995
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,127,0.0129120002190272
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,255,0.015018666783968607
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,255,0.014874666929244995
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,511,0.03107733279466629
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,511,0.0169813334941864
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,1023,0.048698668678601585
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,1023,0.033333333830038704
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,2047,0.0846666693687439
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,2047,0.050144001841545105
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,1,0.019130667050679524
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,1,0.017514667163292568
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,3,0.019098666807015736
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,3,0.01711999997496605
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,7,0.018853332847356796
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,7,0.017594666530688603
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,15,0.017130666722853977
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,15,0.019546666493018467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,31,0.018858666221300762
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,31,0.017525333911180496
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,63,0.018895999838908512
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,63,0.017637333522240322
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,127,0.019280000279347103
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,127,0.01749333366751671
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,255,0.02937600016593933
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,255,0.017173333714405697
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,511,0.04776533444722494
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,511,0.03125333289305369
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,1023,0.08191999793052673
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,1023,0.04854399959246317
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,2047,0.15061333775520325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,2047,0.08400000135103862
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,1,0.015077333897352219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,1,0.015077333897352219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,3,0.015024000157912573
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,3,0.01522133375207583
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,7,0.01516266663869222
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,7,0.015413332730531693
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,15,0.015669333438078564
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,15,0.015360000232855478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,31,0.015029333531856537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,31,0.015119999647140503
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,63,0.015146666516860327
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,63,0.01524266724785169
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,127,0.015061333775520325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,127,0.01534933348496755
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,255,0.01525866612792015
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,255,0.015301333119471868
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,511,0.03145600110292435
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,1,0.019167999426523846
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,1,0.017370666066805523
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,511,0.019706666469573975
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,3,0.019002666076024372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,3,0.01695466662446658
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,7,0.01931200052301089
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,7,0.017418666432301205
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,15,0.019237333287795384
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,15,0.017616000026464462
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,31,0.018895999838908512
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,63,0.01913600042462349
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,31,0.017616000026464462
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,63,0.017055999487638474
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,127,0.019343999524911244
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,127,0.01736533393462499
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,255,0.031152000029881794
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,255,0.01729600007335345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,511,0.048298666874567665
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,511,0.03136533250411352
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,1,0.02736533433198929
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,1,0.025290665527184803
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,3,0.02516266703605652
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,3,0.02758399893840154
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,7,0.027786667148272198
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,7,0.025381334125995636
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,15,0.02773333340883255
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,15,0.02533866713444392
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,31,0.027248000105222065
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,63,0.02829866607983907
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,63,0.025610665480295818
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,31,0.025648000339667004
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,127,0.033039999504884086
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,127,0.02537599951028824
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,255,0.04827733337879181
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,255,0.03192000091075897
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,511,0.05006400247414907
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,511,0.0827893316745758
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,1,0.02146133283774058
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,1,0.01937599976857503
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,3,0.02309866746266683
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,3,0.01924266666173935
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,7,0.021615999440352123
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,7,0.019178666174411774
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,15,0.02309866746266683
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,15,0.01942933350801468
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,31,0.021850667893886566
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,31,0.019445333629846573
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,63,0.023285334308942158
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,63,0.01918399954835574
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,127,0.022154666483402252
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,127,0.01937599976857503
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,255,0.031311998764673867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,255,0.01907733331123988
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,1,0.02977066735426585
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,1,0.02740799884001414
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,3,0.02958400050799052
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,3,0.027189334233601887
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,7,0.029648000995318096
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,7,0.02738133321205775
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,15,0.02951466788848241
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,15,0.027242665489514668
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,31,0.027269333600997925
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,31,0.029701332251230877
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,63,0.029680001238981884
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,63,0.027509334186712902
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,127,0.03552533437808355
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,127,0.027802666028340656
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,255,0.05176533261934916
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,255,0.03565333286921183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,1,0.04995200037956238
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,1,0.04230933388074239
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,3,0.05090666810671488
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,3,0.041893333196640015
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,7,0.04975999891757965
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,7,0.04318933188915253
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,15,0.050623998045921326
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,15,0.04155733436346054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,31,0.049786667029062905
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,31,0.042122667034467064
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,63,0.041797334949175514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,63,0.050853331883748375
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,127,0.0544106662273407
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,1,0.009594666461149851
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,1,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,3,0.00903466654320558
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,127,0.04711466530958811
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,3,0.01097600037852923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,7,0.011098666737476984
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,7,0.011098666737476984
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,15,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,15,0.010714666297038397
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,31,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,63,0.009039999917149544
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,31,0.010762666662534079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,63,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,127,0.010757333288590113
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,255,0.056314667065938316
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,127,0.010816000401973724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,255,0.08536000053087871
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,255,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,511,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,511,0.011221333096424738
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,1023,0.01098666712641716
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,2047,0.014165333161751429
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,2047,0.012928000340859095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,255,0.010768000036478043
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,1023,0.010656000425418219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,4095,0.015674666812022526
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,4095,0.01516266663869222
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,8191,0.01739199956258138
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,8191,0.01711999997496605
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,16383,0.021061333517233532
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,16383,0.03344533344109853
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,32767,0.050053333242734276
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,32767,0.03470933437347412
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,1,0.010869332899649939
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,1,0.010703999549150467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,3,0.010202666744589806
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,3,0.010656000425418219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,7,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,7,0.011194666226704916
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,15,0.010741333166758219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,15,0.010970667004585266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,31,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,31,0.010645333677530289
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,63,0.010938666760921478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,63,0.009754666437705358
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,127,0.0107893335322539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,127,0.009877333417534828
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,255,0.010602666685978571
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,255,0.010928000013033548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,511,0.010842667271693548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,511,0.011157333850860596
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,1023,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,1023,0.011221333096424738
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,2047,0.012784000486135483
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,2047,0.011472000430027643
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,4095,0.013210666676362356
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,4095,0.015146666516860327
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,8191,0.029626667499542236
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,8191,0.015008000036080679
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,16383,0.047450666626294456
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,16383,0.03107200066248576
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,32767,0.0484799991051356
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,32767,0.08094933132330577
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,1,0.009733333562811216
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,1,0.011205332974592844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,3,0.009114666531483332
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,3,0.010138666878143946
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,7,0.009594666461149851
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,7,0.010703999549150467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,15,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,31,0.010853332777818045
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,31,0.009029333169261614
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,15,0.011407999942700068
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,63,0.010847999403874079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,63,0.010666667173306147
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,127,0.009162666896979014
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,255,0.0107893335322539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,255,0.01073066641887029
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,127,0.01110400011142095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,511,0.010666667173306147
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,511,0.011077333241701126
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,1023,0.012831999609867731
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,1023,0.013151999562978745
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,2047,0.012970666090647379
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,2047,0.012981332838535309
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,4095,0.02957333376010259
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,4095,0.0163680004576842
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,8191,0.047925333182017006
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,8191,0.030928000807762146
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,16383,0.08137600123882294
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,16383,0.048245335618654885
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,32767,0.08261866867542267
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,32767,0.1495413382848104
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,1,0.035530666510264076
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,1,0.03133866687615713
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,3,0.03139200061559677
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,3,0.03565333286921183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,7,0.03595733394225439
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,7,0.031386665999889374
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,15,0.030965333183606465
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,15,0.035829332967599235
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,31,0.03153600047032038
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,63,0.035461333890755974
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,31,0.035690667728583016
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,63,0.031397332747777305
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,127,0.0314026673634847
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,1,0.05470933516820272
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,127,0.03923733284076055
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,1,0.04602666695912679
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,3,0.05459733307361603
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,3,0.047653332352638245
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,7,0.053946668903032936
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,7,0.04721599817276001
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,15,0.05389333268006643
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,15,0.04690133531888326
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,31,0.053904001911481224
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,31,0.04606399933497111
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,63,0.05383466680844625
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,127,0.059936001896858215
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,63,0.046240001916885376
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,127,0.05225066840648651
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,1,0.08919999996821086
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,1,0.07850666840871175
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,3,0.09154133001963298
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,3,0.08085333307584126
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,7,0.08903466661771138
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,7,0.07849599917729695
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,15,0.09096533060073853
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,15,0.08096000055472057
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,31,0.09014399846394856
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,31,0.07888533174991608
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,63,0.09153599540392558
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,127,0.09912533561388652
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,63,0.08070933322111766
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,127,0.08251733581225078
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,float16,1,0.06545599798361461
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,1,0.05188799897829691
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,3,0.05182399849096934
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,float16,3,0.06565333406130473
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,float16,7,0.06671466430028279
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,7,0.05186133086681366
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,float16,15,0.06520000100135803
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,float16,31,0.06513600051403046
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,15,0.052101333936055504
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,31,0.05190399785836538
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,float16,63,0.0655626654624939
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,63,0.05212266743183136
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,1,0.08890133102734883
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,float16,1,0.09733866651852925
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,float16,3,0.09622933467229207
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,3,0.08904000123341878
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,float16,7,0.09771200021107991
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,7,0.08898666501045227
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,float16,15,0.09722666939099629
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,15,0.09091200431187947
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,float16,31,0.09794666369756062
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,31,0.08924266695976257
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,float16,63,0.09890666604042053
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,float16,1,0.1694986621538798
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,1,0.1481119990348816
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,63,0.09014399846394856
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,float16,3,0.16517866651217142
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,3,0.14806933204332987
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,float16,7,0.16835200786590576
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,7,0.1476479967435201
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,float16,15,0.16622400283813477
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,15,0.1473973294099172
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,float16,31,0.1698933243751526
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,1,0.010816000401973724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,1,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,3,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,3,0.01157333329319954
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,7,0.010757333288590113
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,7,0.010874666273593903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,15,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,31,0.14825600385665894
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,15,0.010992000500361124
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,31,0.012448000411192576
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,31,0.010608000059922537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,63,0.011034666250149408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,63,0.011120000233252844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,127,0.010965333630641302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,127,0.010768000036478043
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,255,0.010944000134865442
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,255,0.010682666053374609
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,511,0.011194666226704916
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,511,0.011567999919255575
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,1023,0.011322667201360067
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,1023,0.011296000331640244
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,2047,0.013167999684810638
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,2047,0.013466666142145792
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,4095,0.015263999501864115
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,4095,0.014933332800865173
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,float16,63,0.16926399866739908
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,8191,0.029909332593282063
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,8191,0.017717332889636356
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,16383,0.04844800134499868
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,1,0.01071999967098236
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,1,0.010661333799362183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,16383,0.03156266609827677
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,3,0.010944000134865442
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,3,0.010693332801262537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,7,0.009205333267649015
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,7,0.01051733394463857
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,15,0.010751999914646149
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,15,0.010970667004585266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,31,0.011242666592200598
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,31,0.011157333850860596
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,63,0.010794666906197866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,63,0.010832000523805618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,127,0.010757333288590113
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,127,0.011087999989589056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,255,0.010869332899649939
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,255,0.010703999549150467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,511,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,511,0.011354666203260422
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,1023,0.013157332936922709
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,1023,0.012784000486135483
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,63,0.14696000019709268
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,2047,0.013173333058754602
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,2047,0.014917333920796713
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,4095,0.029461334149042766
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,8191,0.047983999053637184
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,8191,0.03145066648721695
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,16383,0.082096000512441
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,4095,0.01603200038274129
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,16383,0.04952533543109894
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,1,0.011328000575304031
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,1,0.010762666662534079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,3,0.010837333897749582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,3,0.010992000500361124
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,7,0.01097600037852923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,7,0.01071999967098236
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,15,0.01097600037852923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,15,0.011183999478816986
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,31,0.01073066641887029
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,31,0.010672000547250112
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,63,0.01097600037852923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,63,0.010965333630641302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,127,0.010853332777818045
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,127,0.010687999427318573
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,255,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,255,0.0107893335322539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,511,0.011071999867757162
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,511,0.011472000430027643
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,1023,0.013072000195582708
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,1023,0.013173333058754602
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,2047,0.029343999922275543
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,2047,0.015013333410024643
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,4095,0.04677333434422811
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,4095,0.03035199890534083
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,8191,0.0805866668621699
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,8191,0.047797332207361855
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,16383,0.14940800269444784
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,16383,0.08138133088747661
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,1,0.010970667004585266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,1,0.010816000401973724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,3,0.010944000134865442
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,3,0.01080000028014183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,7,0.014309333016475042
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,7,0.01101333275437355
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,15,0.0107893335322539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,15,0.011071999867757162
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,31,0.010656000425418219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,31,0.010682666053374609
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,63,0.010591999938090643
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,127,0.010650667051474253
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,127,0.011055999745925268
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,63,0.01098666712641716
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,255,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,255,0.010709332923094431
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,511,0.011071999867757162
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,1023,0.011760000139474869
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,511,0.011322667201360067
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,1023,0.012560000022252401
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,2047,0.013034666577974955
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,2047,0.013162666310866674
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,8191,0.031317333380381264
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,4095,0.01523200049996376
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,4095,0.01522133375207583
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,8191,0.018197332819302876
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,16383,0.048437332113583885
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,16383,0.032085334261258446
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,1,0.009306666751702627
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,1,0.0107893335322539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,3,0.010762666662534079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,3,0.011039999624093374
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,7,0.010154666379094124
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,7,0.011215999722480774
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,15,0.010778666784365972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,15,0.010714666297038397
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,63,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,31,0.009466666728258133
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,31,0.010954666882753372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,63,0.01118933285276095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,127,0.010682666053374609
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,127,0.010874666273593903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,255,0.010853332777818045
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,255,0.010965333630641302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,511,0.010954666882753372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,511,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,1023,0.012869333227475485
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,1023,0.012815999488035837
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,2047,0.012954667210578918
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,2047,0.013056000073750814
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,4095,0.029696000119050343
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,4095,0.017103999853134155
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,8191,0.04831466575463613
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,8191,0.03146133323510488
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,16383,0.08319999774297078
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,16383,0.04967466493447622
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,1,0.010842667271693548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,1,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,3,0.010714666297038397
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,3,0.011061333119869232
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,7,0.01102399950226148
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,7,0.010911999891201654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,15,0.011152000476916632
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,15,0.010847999403874079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,31,0.01080000028014183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,31,0.010805333654085795
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,63,0.01091733326514562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,63,0.010709332923094431
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,127,0.010634666929642359
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,127,0.010928000013033548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,255,0.010821333775917688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,255,0.011050666371981302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,511,0.010965333630641302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,511,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,1023,0.013269333789745966
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,1023,0.013007999708255133
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,2047,0.01523200049996376
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,2047,0.029333333174387615
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,4095,0.0469760000705719
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,8191,0.047775998711586
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,4095,0.029989334444204967
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,8191,0.08098133405049641
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,16383,0.1495680014292399
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,1,0.011866666376590729
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,1,0.010970667004585266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,16383,0.08258133133252461
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,3,0.013232000172138214
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,3,0.01128000020980835
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,7,0.01145600030819575
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,7,0.011221333096424738
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,15,0.011045332998037338
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,15,0.011050666371981302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,31,0.011231999844312668
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,31,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,63,0.011488000551859537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,63,0.012351999680201212
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,127,0.010928000013033548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,127,0.011061333119869232
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,255,0.011215999722480774
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,255,0.012981332838535309
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,511,0.01443733274936676
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,511,0.012906666845083237
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,1023,0.029296000798543293
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,1023,0.015189333508412043
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,2047,0.029663999875386555
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,4095,0.04789333542188009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,4095,0.08699733018875122
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,2047,0.04760533571243286
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,8191,0.08338666955629985
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,8191,0.1516213317712148
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,1,0.008869333192706108
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,1,0.009178666397929192
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,16383,0.2904106577237447
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,3,0.008858666444818178
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,3,0.009328000247478485
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,7,0.010714666297038397
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,7,0.009685333197315535
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,15,0.009093333035707474
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,16383,0.15363732973734537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,15,0.010687999427318573
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,31,0.008997333546479544
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,31,0.01028266673286756
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,63,0.010687999427318573
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,127,0.009279999881982803
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,127,0.01055466632048289
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,63,0.011253333340088526
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,255,0.010874666273593903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,255,0.009002666920423508
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,511,0.011034666250149408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,511,0.011781333635250727
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,1023,0.011168000598748526
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,1023,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,2047,0.01109333336353302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,2047,0.011551999797423681
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,4095,0.01128000020980835
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,4095,0.01119999960064888
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,8191,0.015082667271296183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,16383,0.019248000035683315
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,32767,0.027263998985290527
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,16383,0.0191040001809597
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,8191,0.015583999454975128
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,32767,0.027514666318893433
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,65535,0.03138133386770884
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,65535,0.03133333226044973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,1,0.01020800011853377
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,1,0.010650667051474253
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,3,0.008912000184257826
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,3,0.008869333192706108
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,7,0.009583999713261923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,7,0.010586666564146677
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,15,0.009082666908701261
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,15,0.00895999992887179
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,31,0.010575999816258749
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,63,0.009109333157539368
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,31,0.010255999863147736
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,63,0.009114666531483332
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,127,0.009279999881982803
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,127,0.009178666397929192
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,255,0.008949333180983862
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,255,0.011125333607196808
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,511,0.010693332801262537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,511,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,1023,0.010784000158309937
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,1023,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,2047,0.011109333485364914
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,2047,0.011152000476916632
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,4095,0.010869332899649939
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,4095,0.01156266654531161
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,8191,0.015306666493415833
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,8191,0.015216000378131866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,16383,0.019130667050679524
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,16383,0.01941866676012675
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,32767,0.021295999487241108
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,32767,0.021509334444999695
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,65535,0.023130667706330616
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,1,0.010640000303586325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,65535,0.02348800003528595
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,1,0.008832000195980072
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,3,0.009045333291093508
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,3,0.01081066702802976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,7,0.008965333302815756
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,7,0.010847999403874079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,15,0.009130666653315226
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,15,0.010549332946538925
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,31,0.008997333546479544
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,31,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,63,0.008896000062425932
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,63,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,127,0.009194666519761086
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,127,0.010832000523805618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,255,0.00927466650803884
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,255,0.008976000050703684
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,511,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,511,0.010938666760921478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,1023,0.0102613332370917
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,1023,0.010762666662534079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,2047,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,2047,0.01118933285276095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,4095,0.010938666760921478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,4095,0.011424000064531961
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,8191,0.015669333438078564
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,8191,0.015072000523408255
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,16383,0.017935999979575474
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,16383,0.017242666333913803
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,32767,0.01953599974513054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,32767,0.019296000401178997
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,65535,0.036015999813874565
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,65535,0.02117866774400075
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,1,0.010645333677530289
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,1,0.009648000200589498
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,3,0.009136000027259191
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,3,0.01099733387430509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,7,0.009226666763424873
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,7,0.009317333499590555
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,15,0.009488000224033991
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,15,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,31,0.009119999905427298
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,31,0.01080000028014183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,63,0.011002667248249054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,127,0.009189333145817121
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,63,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,127,0.010741333166758219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,255,0.009354666496316591
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,255,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,511,0.01071999967098236
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,511,0.011354666203260422
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,1023,0.011087999989589056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,1023,0.010784000158309937
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,2047,0.011055999745925268
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,2047,0.011567999919255575
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,4095,0.012885333349307379
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,4095,0.013370666652917862
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,8191,0.014874666929244995
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,8191,0.015413332730531693
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,16383,0.017093333105246227
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,16383,0.01709866647919019
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,32767,0.03306666761636734
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,32767,0.019199999670187633
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,65535,0.050767997900644936
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,1,0.009050666665037474
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,65535,0.03549333413441976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,1,0.011274666835864386
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,3,0.00956266683836778
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,3,0.008885333314538002
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,7,0.010725333044926325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,7,0.010970667004585266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,15,0.008863999818762144
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,15,0.010874666273593903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,31,0.010869332899649939
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,31,0.011264000087976456
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,63,0.009178666397929192
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,63,0.010288000106811523
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,127,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,127,0.010938666760921478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,255,0.008885333314538002
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,255,0.010911999891201654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,511,0.011306667079528173
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,511,0.01101333275437355
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,1023,0.010794666906197866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,1023,0.010938666760921478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,2047,0.011226666470368704
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,4095,0.01108266661564509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,2047,0.01146666705608368
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,4095,0.01121066634853681
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,8191,0.01509333277742068
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,8191,0.01568000018596649
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,16383,0.019466667125622433
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,16383,0.021183999876181286
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,32767,0.02317333221435547
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,32767,0.023178666830062866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,65535,0.024218666056791942
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,65535,0.025370667378107708
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,1,0.00890666681031386
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,1,0.010709332923094431
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,3,0.00884799969693025
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,3,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,7,0.009082666908701261
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,15,0.009077333534757296
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,7,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,15,0.010565333068370819
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,31,0.008896000062425932
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,31,0.009039999917149544
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,63,0.009322666873534521
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,63,0.011402666568756104
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,127,0.00927466650803884
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,127,0.010725333044926325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,255,0.011007999380429586
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,255,0.008869333192706108
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,511,0.010837333897749582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,511,0.011237333218256632
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,1023,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,1023,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,2047,0.010944000134865442
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,4095,0.010911999891201654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,2047,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,4095,0.011312000453472137
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,8191,0.015013333410024643
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,8191,0.01515199989080429
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,16383,0.017722666263580322
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,16383,0.016949333250522614
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,32767,0.019258666783571243
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,32767,0.019082666685183842
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,65535,0.035530666510264076
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,65535,0.020992000897725422
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,1,0.009813333551088968
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,1,0.009183999771873156
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,3,0.008997333546479544
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,3,0.010698666175206503
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,7,0.009189333145817121
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,7,0.009130666653315226
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,15,0.010666667173306147
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,15,0.009093333035707474
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,31,0.008826666822036108
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,31,0.01080000028014183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,63,0.009205333267649015
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,63,0.00895999992887179
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,127,0.011125333607196808
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,127,0.00903466654320558
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,255,0.009136000027259191
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,255,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,511,0.01091733326514562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,511,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,1023,0.011050666371981302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,1023,0.011002667248249054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,2047,0.011050666371981302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,2047,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,4095,0.012949333836634954
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,4095,0.013088000317414602
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,8191,0.014970666418472925
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,16383,0.017103999853134155
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,16383,0.01701333373785019
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,8191,0.016234666109085083
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,32767,0.018858666221300762
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,32767,0.03249600032965342
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,65535,0.051455999414126076
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,65535,0.035989334185918175
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,1,0.010672000547250112
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,1,0.009685333197315535
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,3,0.009557333464423815
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,7,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,7,0.009775999933481216
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,3,0.01138666644692421
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,15,0.009557333464423815
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,15,0.010842667271693548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,31,0.009269333134094873
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,63,0.010703999549150467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,31,0.010842667271693548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,63,0.009125333279371262
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,127,0.01073066641887029
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,127,0.0107893335322539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,255,0.009301333377758661
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,255,0.010112000008424124
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,511,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,511,0.010746666540702185
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,1023,0.010794666906197866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,1023,0.011264000087976456
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,2047,0.013232000172138214
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,2047,0.012453333785136541
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,4095,0.01322666679819425
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,8191,0.014938666174809137
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,4095,0.014069333672523499
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,8191,0.016927999754746754
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,16383,0.01736533393462499
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,16383,0.031370667119820915
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,32767,0.03320533285538355
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,32767,0.05039466420809428
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,65535,0.05009066561857859
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,65535,0.0825493335723877
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,1,0.011066666493813196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,3,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,1,0.01138666644692421
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,3,0.011018666128317514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,7,0.010645333677530289
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,7,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,15,0.010832000523805618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,15,0.011402666568756104
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,31,0.010778666784365972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,31,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,63,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,127,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,63,0.010944000134865442
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,127,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,255,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,255,0.010741333166758219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,511,0.013045333325862885
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,511,0.011776000261306763
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,1023,0.012901333471139273
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,1023,0.013317332913478216
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,2047,0.01509333277742068
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,2047,0.015029333531856537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,8191,0.049216002225875854
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,4095,0.016976000120242436
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,4095,0.031712000568707786
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,8191,0.03137599925200144
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,1,0.01073066641887029
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,1,0.01118933285276095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,3,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,3,0.010874666273593903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,7,0.010575999816258749
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,15,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,15,0.010944000134865442
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,7,0.011296000331640244
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,31,0.01119999960064888
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,31,0.010847999403874079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,63,0.01099733387430509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,127,0.010693332801262537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,63,0.011098666737476984
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,127,0.010837333897749582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,255,0.010762666662534079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,255,0.010741333166758219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,511,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,511,0.010992000500361124
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,1023,0.01302933320403099
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,1023,0.012906666845083237
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,4095,0.0476693312327067
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,2047,0.01498666654030482
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,2047,0.03012799968322118
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,4095,0.03033600002527237
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,8191,0.08422933022181193
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,1,0.012661332885424295
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,8191,0.047824000318845115
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,1,0.011407999942700068
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,3,0.01227733368674914
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,3,0.012213333199421564
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,7,0.012256000190973282
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,15,0.012698666503032049
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,7,0.011413333316644033
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,15,0.01258133351802826
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,31,0.012367999802033106
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,31,0.010992000500361124
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,63,0.01314666618903478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,63,0.011194666226704916
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,127,0.0116799995303154
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,127,0.011322667201360067
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,255,0.012981332838535309
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,255,0.011098666737476984
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,511,0.015002666662136713
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,511,0.013183999806642532
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,1023,0.029738667110602062
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,1023,0.015376000354687372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,2047,0.04775999983151754
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,2047,0.030778666337331135
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,4095,0.04957866668701172
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,4095,0.082997332016627
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,8191,0.08497066299120586
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,8191,0.1521440049012502
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,1,0.014965333044528961
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,1,0.013306666165590286
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,3,0.014752000570297241
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,3,0.013189333180586496
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,7,0.014943999548753103
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,15,0.012997332960367203
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,7,0.013189333180586496
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,15,0.015013333410024643
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,31,0.01302933320403099
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,63,0.012879999975363413
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,31,0.01516266663869222
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,63,0.015018666783968607
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,127,0.013050666699806849
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,127,0.01463466634353002
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,255,0.014975999792416891
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,255,0.013162666310866674
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,511,0.02939733366171519
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,511,0.01711999997496605
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,1023,0.048026666045188904
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,1023,0.03186133255561193
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,2047,0.049546668926874794
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,4095,0.1545866628487905
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,4095,0.08507200082143147
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,2047,0.08278400202592213
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,1,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,1,0.010842667271693548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,3,0.010672000547250112
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,3,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,7,0.010832000523805618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,7,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,15,0.010687999427318573
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,8191,0.15507733821868896
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,15,0.010559999694426855
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,8191,0.29474665721257526
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,31,0.009946666657924652
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,31,0.011034666250149408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,63,0.011018666128317514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,127,0.010911999891201654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,63,0.010928000013033548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,127,0.010874666273593903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,255,0.010661333799362183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,255,0.010885333021481832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,511,0.010629333555698395
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,1023,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,511,0.011157333850860596
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,1023,0.010725333044926325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,2047,0.01102399950226148
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,2047,0.011237333218256632
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,4095,0.011226666470368704
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,4095,0.01118933285276095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,8191,0.01709866647919019
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,8191,0.01709866647919019
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,16383,0.019093333433071773
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,16383,0.01889066646496455
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,32767,0.021397332350413006
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,32767,0.01918399954835574
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,65535,0.035786665976047516
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,65535,0.023034666975339253
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,1,0.008896000062425932
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,3,0.009392000113924345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,1,0.011109333485364914
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,3,0.009136000027259191
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,7,0.010565333068370819
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,15,0.009056000038981438
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,7,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,15,0.010911999891201654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,31,0.009130666653315226
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,31,0.010944000134865442
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,63,0.01071999967098236
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,63,0.009008000294367472
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,127,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,127,0.010911999891201654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,255,0.008933333059151968
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,255,0.01102399950226148
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,511,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,511,0.011055999745925268
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,1023,0.010928000013033548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,1023,0.011141333729028702
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,2047,0.0107893335322539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,2047,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,4095,0.013866666704416275
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,4095,0.013167999684810638
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,8191,0.014943999548753103
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,8191,0.014938666174809137
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,16383,0.017456000049908955
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,16383,0.016906666258970898
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,32767,0.033402666449546814
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,32767,0.01714133347074191
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,65535,0.03417599946260452
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,65535,0.051072001457214355
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,1,0.00949866697192192
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,1,0.009082666908701261
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,3,0.010026666646202406
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,3,0.010954666882753372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,7,0.00922133338948091
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,7,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,15,0.009258666386206945
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,15,0.010837333897749582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,31,0.00903466654320558
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,31,0.010928000013033548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,63,0.00978133330742518
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,63,0.010832000523805618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,127,0.009216000015536943
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,127,0.010687999427318573
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,255,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,511,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,255,0.01080000028014183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,511,0.01118933285276095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,1023,0.010714666297038397
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,1023,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,2047,0.012991999586423239
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,2047,0.013002666334311167
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,4095,0.012965332716703415
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,4095,0.015002666662136713
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,8191,0.014943999548753103
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,8191,0.015429332852363586
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,16383,0.018485333770513535
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,16383,0.0314026673634847
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,32767,0.048725331823031105
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,32767,0.03380800038576126
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,65535,0.04983466863632202
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,65535,0.08353066444396973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,1,0.009114666531483332
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,1,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,3,0.010687999427318573
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,3,0.008992000172535578
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,7,0.009173333023985228
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,7,0.010490667074918747
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,15,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,15,0.009599999835093817
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,31,0.00884799969693025
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,31,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,63,0.010250666489203772
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,63,0.010677333921194077
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,127,0.00933333362142245
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,127,0.010816000401973724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,255,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,255,0.010154666379094124
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,511,0.011125333607196808
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,511,0.01119999960064888
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,1023,0.011370666325092316
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,1023,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,2047,0.012362666428089142
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,2047,0.012917333592971167
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,4095,0.01498666654030482
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,4095,0.013183999806642532
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,8191,0.029648000995318096
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,8191,0.01526933287580808
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,16383,0.04800533254941305
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,16383,0.03108799954255422
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,32767,0.08096000055472057
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,32767,0.04809600114822388
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,65535,0.08238400022188823
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,65535,0.15017599860827127
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,1,0.011173332730929056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,1,0.011034666250149408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,3,0.011130666981140772
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,3,0.011274666835864386
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,7,0.010869332899649939
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,15,0.011087999989589056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,7,0.01118933285276095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,15,0.01102399950226148
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,31,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,31,0.011274666835864386
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,63,0.011215999722480774
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,63,0.011045332998037338
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,127,0.010698666175206503
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,127,0.01119999960064888
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,255,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,511,0.011141333729028702
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,255,0.011087999989589056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,511,0.013269333789745966
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,1023,0.015119999647140503
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,1023,0.01292266696691513
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,2047,0.029498666524887085
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,2047,0.017514667163292568
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,4095,0.04806933303674062
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,4095,0.03155199935038885
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,1,0.011152000476916632
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,1,0.012298667182525
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,3,0.012117333710193634
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,3,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,7,0.012437333663304647
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,7,0.012639999389648438
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,15,0.012549333274364471
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,15,0.011018666128317514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,31,0.012805332740147909
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,31,0.012554666648308435
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,63,0.012485332787036896
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,63,0.010944000134865442
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,127,0.012661332885424295
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,127,0.012874666601419449
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,255,0.012879999975363413
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,255,0.011098666737476984
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,511,0.015082667271296183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,511,0.013424000392357508
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,1023,0.029968000948429108
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,1023,0.015978666643301647
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,2047,0.04779199759165446
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,4095,0.050399998823801674
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,4095,0.08482666810353597
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,2047,0.03146133323510488
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,1,0.015061333775520325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,1,0.01320533330241839
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,3,0.014842666685581207
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,3,0.013376000026861826
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,7,0.01498666654030482
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,7,0.013013333082199097
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,15,0.015125333021084467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,15,0.013487999637921652
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,31,0.015066667149464289
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,31,0.013023999830087027
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,63,0.014837333311637243
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,63,0.01340266689658165
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,127,0.013951999445756277
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,127,0.01309866706530253
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,255,0.015034666905800501
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,255,0.013232000172138214
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,511,0.029733332494894665
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,511,0.01720000058412552
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,1023,0.04830400149027506
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,2047,0.0842186709245046
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,2047,0.05011733373006185
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,1023,0.03180266668399175
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,4095,0.15598932902018228
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,4095,0.08659733335177104
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,1,0.019744000087181728
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,1,0.017498667041460674
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,3,0.01887999971707662
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,3,0.016997333616018295
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,7,0.019199999670187633
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,7,0.017429333180189133
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,15,0.017103999853134155
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,15,0.019343999524911244
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,31,0.019274666905403137
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,31,0.01746133342385292
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,63,0.019130667050679524
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,63,0.01743999992807706
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,127,0.019413333386182785
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,127,0.017418666432301205
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,255,0.029343999922275543
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,255,0.016949333250522614
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,511,0.04789333542188009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,1023,0.0813920001188914
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,511,0.033359999457995095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,2047,0.15017599860827127
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,1023,0.04915733138720194
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,2047,0.08286400139331818
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,4095,0.1520906686782837
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,4095,0.2866026759147644
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,1,0.012896000097195307
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,1,0.013269333789745966
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,3,0.012975999464591345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,7,0.012842666357755661
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,7,0.013125333935022354
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,3,0.013050666699806849
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,15,0.012938667088747025
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,15,0.012975999464591345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,31,0.01293333371480306
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,31,0.012986666212479273
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,63,0.012853333105643591
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,63,0.013072000195582708
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,127,0.013189333180586496
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,127,0.013199999928474426
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,255,0.012986666212479273
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,255,0.01302933320403099
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,511,0.015365333606799444
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,511,0.014912000546852747
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,1023,0.0308693324526151
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,1023,0.017103999853134155
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,1,0.014933332800865173
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,1,0.015072000523408255
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,3,0.015018666783968607
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,3,0.013088000317414602
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,7,0.014997333288192749
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,7,0.014970666418472925
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,15,0.014736000448465347
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,15,0.012970666090647379
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,31,0.01488000030318896
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,31,0.01516266663869222
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,63,0.014720000326633453
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,63,0.01293333371480306
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,127,0.01492799942692121
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,255,0.014853333433469137
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,127,0.014831999937693277
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,255,0.013056000073750814
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,511,0.03129599988460541
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,511,0.01695466662446658
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,1023,0.049626668294270836
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,1023,0.03148799886306127
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,1,0.018837332725524902
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,1,0.01709866647919019
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,3,0.019589333484570186
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,3,0.017029333859682083
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,7,0.01729600007335345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,7,0.018863999595244724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,15,0.01724799970785777
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,15,0.01937599976857503
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,31,0.019029332945744198
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,31,0.01743999992807706
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,63,0.019093333433071773
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,63,0.020453333854675293
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,127,0.019082666685183842
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,127,0.017573333034912746
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,255,0.02956799914439519
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,255,0.017594666530688603
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,511,0.04763199885686239
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,511,0.03160533308982849
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,1023,0.08262399832407634
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,1,0.027280000348885853
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,1,0.025386666258176167
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,1023,0.049957334995269775
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,3,0.025397333006064098
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,7,0.027637332677841187
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,3,0.02734400083621343
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,7,0.025381334125995636
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,15,0.025279998779296875
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,15,0.027317332724730175
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,31,0.027274665733178455
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,31,0.025392000873883564
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,63,0.025226667523384094
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,63,0.029205332199732464
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,127,0.03194133440653483
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,127,0.02569066733121872
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,255,0.04771199822425842
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,255,0.03249600032965342
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,511,0.0828906645377477
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,1023,0.15055466691652933
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,1023,0.0842080016930898
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,511,0.04994133114814758
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,1,0.015311999867359797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,1,0.014997333288192749
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,3,0.016970666746298473
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,3,0.015189333508412043
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,7,0.015322666615247726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,7,0.015072000523408255
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,15,0.01700266698996226
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,15,0.01534933348496755
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,31,0.016261332978804905
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,31,0.01516266663869222
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,63,0.01704000060757001
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,63,0.01505600040157636
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,127,0.015247999380032221
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,127,0.015290666371583939
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,255,0.015018666783968607
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,511,0.0314026673634847
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,511,0.01934933289885521
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,255,0.017445333302021027
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,1,0.019018666197856266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,1,0.017055999487638474
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,3,0.019141333798567455
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,3,0.017632000148296356
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,7,0.019061333189407986
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,7,0.01735466718673706
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,15,0.019354666272799175
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,31,0.019146667172511418
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,15,0.017525333911180496
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,31,0.017093333105246227
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,63,0.019306667149066925
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,63,0.017045332739750545
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,127,0.019674666225910187
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,127,0.01706133286158244
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,255,0.03129599988460541
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,255,0.017029333859682083
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,511,0.031370667119820915
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,511,0.049546668926874794
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,1,0.02956799914439519
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,1,0.02741333345572154
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,3,0.029311999678611755
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,3,0.027530667682488758
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,7,0.0295413335164388
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,7,0.027221334477265675
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,15,0.029440000653266907
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,15,0.02737066646416982
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,31,0.0295413335164388
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,31,0.02718399961789449
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,63,0.02932800104220708
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,63,0.02754666656255722
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,127,0.03547733277082443
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,127,0.027322667340437572
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,255,0.0497920016447703
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,255,0.03565333286921183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,511,0.08482666810353597
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,511,0.052560001611709595
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,1,0.05029866596062978
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,1,0.04200000067551931
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,3,0.050144001841545105
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,3,0.041663999358812966
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,7,0.05012799799442291
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,7,0.04222933451334635
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,15,0.050160000721613564
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,15,0.04178133110205332
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,31,0.05026666820049286
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,31,0.04207466542720795
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,63,0.04173333446184794
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,63,0.05106666684150696
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,127,0.05423999826113383
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,127,0.046767999728520714
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,255,0.0849173367023468
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,255,0.0552106648683548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,1,0.010709332923094431
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,1,0.010746666540702185
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,3,0.010597333312034607
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,3,0.010682666053374609
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,511,0.1548799971739451
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,7,0.010629333555698395
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,7,0.010410666465759277
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,15,0.008832000195980072
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,31,0.011221333096424738
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,31,0.009216000015536943
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,63,0.010741333166758219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,63,0.00915733352303505
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,127,0.010591999938090643
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,127,0.009941333283980688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,255,0.010762666662534079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,15,0.008853333070874214
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,255,0.010954666882753372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,511,0.011050666371981302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,511,0.011071999867757162
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,1023,0.01108266661564509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,1023,0.011247999966144562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,2047,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,2047,0.011120000233252844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,4095,0.016501333564519882
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,8191,0.016970666746298473
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,4095,0.015002666662136713
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,8191,0.01695466662446658
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,16383,0.019189332922299702
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,16383,0.017050666113694508
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,32767,0.03403199960788091
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,32767,0.02124800036350886
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,65535,0.05193600058555603
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,65535,0.035749333600203194
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,1,0.009914666414260864
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,1,0.008890666688481966
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,3,0.009642666826645533
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,3,0.01097600037852923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,7,0.009082666908701261
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,7,0.010431999961535135
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,15,0.009194666519761086
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,15,0.010645333677530289
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,31,0.008933333059151968
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,511,0.08770133058230083
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,31,0.010581333190202713
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,63,0.009136000027259191
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,63,0.010751999914646149
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,127,0.008986666798591614
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,255,0.011215999722480774
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,255,0.008992000172535578
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,511,0.010821333775917688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,511,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,1023,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,1023,0.010938666760921478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,127,0.010469333579142889
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,2047,0.012810666114091873
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,2047,0.012965332716703415
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,4095,0.014869333555301031
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,4095,0.013253333667914072
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,8191,0.016000000139077503
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,8191,0.015237333873907724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,16383,0.03163733333349228
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,16383,0.017103999853134155
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,32767,0.05018133421738943
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,32767,0.03349333256483078
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,65535,0.08497066299120586
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,65535,0.050383999943733215
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,1,0.010735999792814255
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,1,0.010874666273593903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,3,0.010725333044926325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,3,0.010911999891201654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,7,0.010816000401973724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,7,0.010928000013033548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,15,0.010703999549150467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,31,0.010981333752473196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,31,0.010821333775917688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,15,0.01091733326514562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,63,0.010928000013033548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,127,0.010874666273593903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,63,0.010677333921194077
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,127,0.011381333072980246
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,255,0.010474666953086853
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,255,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,511,0.011125333607196808
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,511,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,1023,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,1023,0.011173332730929056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,2047,0.012810666114091873
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,2047,0.011477333803971609
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,4095,0.013178666432698568
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,4095,0.014602666099866232
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,8191,0.029824001093705494
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,8191,0.01553600033124288
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,16383,0.029669334491093952
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,16383,0.04765866696834564
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,32767,0.08135466774304707
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,32767,0.04837333162625631
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,65535,0.08429333567619324
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,65535,0.15030399958292642
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,1,0.010069333637754122
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,1,0.010784000158309937
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,3,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,3,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,7,0.010911999891201654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,7,0.011317333827416102
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,15,0.010559999694426855
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,15,0.010853332777818045
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,31,0.010938666760921478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,31,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,63,0.01097600037852923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,63,0.01101333275437355
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,127,0.010874666273593903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,127,0.010703999549150467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,255,0.010575999816258749
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,255,0.01091733326514562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,511,0.01081066702802976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,511,0.011146667102972666
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,1023,0.01091733326514562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,1023,0.012842666357755661
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,2047,0.01320533330241839
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,2047,0.013338666409254074
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,4095,0.02938133229811986
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,4095,0.01534933348496755
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,8191,0.03138133386770884
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,16383,0.048026666045188904
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,8191,0.04714133342107137
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,16383,0.08124800026416779
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,32767,0.0832479993502299
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,32767,0.15037866433461508
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,65535,0.15031466881434122
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,65535,0.2863413294156392
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,1,0.01926400015751521
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,1,0.023018665611743927
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,3,0.021114667256673176
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,3,0.019285333653291065
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,7,0.021333334346612293
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,7,0.020432000358899433
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,15,0.02186133215824763
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,15,0.019493332753578823
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,31,0.02142400046189626
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,63,0.02139200021823247
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,31,0.01998399943113327
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,127,0.021488000949223835
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,255,0.03152533372243246
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,127,0.02027200038234393
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,63,0.019744000087181728
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,1,0.029232000311215717
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,255,0.02109866589307785
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,1,0.02762666592995326
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,3,0.02976000060637792
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,3,0.02773333340883255
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,7,0.029509333272775013
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,7,0.027450665831565857
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,15,0.029722665747006733
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,15,0.027306665976842243
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,31,0.02961066613594691
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,31,0.02741333345572154
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,63,0.029834667841593426
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,63,0.02718399961789449
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,127,0.035445332527160645
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,127,0.028922667105992634
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,255,0.05194133520126343
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,255,0.03562666724125544
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,1,0.05384000142415365
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,1,0.045978665351867676
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,3,0.04494399825731913
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,7,0.05454933146635691
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,3,0.05431999762852987
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,7,0.045893331368764244
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,15,0.053599998354911804
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,15,0.04586133360862732
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,31,0.045978665351867676
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,31,0.05449066559473673
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,63,0.04580800235271454
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,63,0.054197331269582115
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,127,0.060559997955958046
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,127,0.05203199883302053
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,255,0.06128533184528351
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,255,0.09098666906356812
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,1,0.09100799759229024
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,1,0.07983466486136119
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,3,0.08894399801890056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,3,0.08056533336639404
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,7,0.09098666906356812
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,7,0.07950399816036224
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,15,0.08966400225957234
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,15,0.0800799975792567
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,31,0.09113599856694539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,31,0.0804746647675832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,63,0.09132267038027446
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,63,0.07970133423805237
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,127,0.0995360016822815
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,127,0.08261333405971527
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,255,0.1600213348865509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,255,0.09918933113416036
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,1,0.03573866685231527
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,1,0.031727999448776245
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,3,0.035530666510264076
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,3,0.031146667897701263
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,7,0.035562666753927864
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,7,0.03170666595300039
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,15,0.03563733398914337
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,15,0.03145600110292435
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,31,0.03570666660865148
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,31,0.03155199935038885
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,63,0.03555200000603994
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,63,0.031104000906149547
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,127,0.0395359992980957
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,1,0.052986666560173035
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,1,0.04380266865094503
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,127,0.032101333141326904
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,3,0.05314133564631144
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,7,0.052655999859174095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,7,0.0444160004456838
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,3,0.04417066772778829
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,15,0.053541332483291626
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,15,0.04474666714668274
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,31,0.05384000142415365
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,31,0.04409066836039225
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,63,0.05316799879074097
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,127,0.04981866478919983
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,63,0.04379733403523763
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,1,0.09709866841634114
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,127,0.06025066475073496
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,1,0.08789867162704468
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,3,0.09813867012659709
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,3,0.08896533648173015
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,7,0.09875733653704326
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,7,0.08814400434494019
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,15,0.09779199957847595
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,15,0.08899733424186707
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,31,0.09915199875831604
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,31,0.08807999889055888
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,63,0.09759466846783955
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,63,0.08889066179593404
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,127,0.1081813375155131
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,127,0.09084266424179077
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,1,0.1658506691455841
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,1,0.14702399571736655
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,3,0.16714666287104288
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,3,0.14437333742777506
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,7,0.16615999738375345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,7,0.1464959979057312
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,15,0.14454399545987448
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,15,0.16796799500783285
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,31,0.1671733260154724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,31,0.14689067006111145
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,63,0.16921599706013998
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,1,0.009061333412925402
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,63,0.14662933349609375
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,1,0.010693332801262537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,3,0.01101333275437355
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,7,0.008986666798591614
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,3,0.011039999624093374
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,7,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,15,0.010837333897749582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,15,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,31,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,31,0.010703999549150467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,63,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,63,0.010768000036478043
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,127,0.010970667004585266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,127,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,255,0.010703999549150467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,255,0.010698666175206503
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,511,0.010842667271693548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,511,0.011258666714032492
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,1023,0.011002667248249054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,1023,0.011194666226704916
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,2047,0.013082666943470636
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,2047,0.013301332791646322
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,4095,0.015008000036080679
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,8191,0.01704000060757001
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,4095,0.015279999623696009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,8191,0.016821333517630894
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,16383,0.03345600018898646
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,127,0.18453333775202432
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,16383,0.01945066700379054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,32767,0.05021866659323374
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,32767,0.033904001116752625
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,1,0.010698666175206503
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,3,0.00890666681031386
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,7,0.009173333023985228
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,3,0.010709332923094431
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,7,0.009018666421373686
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,1,0.009690666571259499
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,15,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,15,0.01091733326514562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,31,0.009103999783595404
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,31,0.010741333166758219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,63,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,63,0.011045332998037338
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,127,0.010911999891201654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,127,0.010928000013033548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,255,0.00996800015370051
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,255,0.010693332801262537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,511,0.010954666882753372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,511,0.011050666371981302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,1023,0.01126933346192042
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,1023,0.010970667004585266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,2047,0.012975999464591345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,2047,0.012847999731699625
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,4095,0.013760000467300415
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,4095,0.012975999464591345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,8191,0.015168000012636185
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,8191,0.03036266565322876
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,16383,0.047877331574757896
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,16383,0.030437332888444264
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,32767,0.08362666765848796
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,32767,0.0488319993019104
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,1,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,1,0.010618666807810465
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,3,0.01028266673286756
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,3,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,7,0.010746666540702185
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,7,0.011194666226704916
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,15,0.01071999967098236
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,15,0.010666667173306147
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,31,0.010874666273593903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,31,0.010768000036478043
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,127,0.1520960032939911
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,63,0.011018666128317514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,63,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,127,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,127,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,255,0.010784000158309937
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,511,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,511,0.011045332998037338
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,1023,0.012800000607967377
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,1023,0.01292266696691513
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,2047,0.013530666629473368
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,255,0.010640000303586325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,2047,0.01332266628742218
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,4095,0.029711998999118805
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,4095,0.016447999825080235
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,8191,0.04782933493455251
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,8191,0.03146133323510488
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,16383,0.08136000235875447
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,16383,0.049653331438700356
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,32767,0.0836906631787618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,1,0.01098666712641716
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,32767,0.15060266852378845
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,1,0.011050666371981302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,3,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,3,0.010725333044926325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,7,0.010911999891201654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,7,0.010885333021481832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,15,0.010837333897749582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,15,0.010853332777818045
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,31,0.010832000523805618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,31,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,63,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,63,0.01091733326514562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,127,0.011045332998037338
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,127,0.010805333654085795
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,255,0.010698666175206503
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,255,0.010821333775917688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,511,0.011114666859308878
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,511,0.011274666835864386
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,1023,0.013343999783198038
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,1023,0.01310933381319046
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,2047,0.029733332494894665
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,4095,0.029370665550231934
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,4095,0.04734399914741516
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,2047,0.015168000012636185
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,8191,0.0811413327852885
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,8191,0.047877331574757896
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,16383,0.1499626636505127
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,16383,0.08207466701666515
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,1,0.010533332824707031
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,32767,0.28649600346883136
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,32767,0.15037866433461508
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,1,0.011226666470368704
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,3,0.010778666784365972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,7,0.010645333677530289
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,3,0.01073066641887029
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,7,0.011029332876205444
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,15,0.010570666442314783
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,15,0.010853332777818045
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,31,0.010661333799362183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,31,0.01118933285276095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,63,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,127,0.010762666662534079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,63,0.01062400018175443
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,255,0.011002667248249054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,255,0.011066666493813196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,127,0.01099733387430509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,511,0.011120000233252844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,511,0.01080000028014183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,1023,0.012794667234023413
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,1023,0.012186666329701742
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,2047,0.012965332716703415
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,2047,0.012901333471139273
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,4095,0.015066667149464289
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,4095,0.015072000523408255
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,8191,0.03164266546567281
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,8191,0.017082666357358296
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,16383,0.04976533353328705
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,1,0.010602666685978571
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,1,0.009312000125646591
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,16383,0.03198933353026708
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,3,0.010703999549150467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,3,0.010741333166758219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,7,0.01091733326514562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,7,0.0103946669648091
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,15,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,31,0.010954666882753372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,15,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,31,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,63,0.010666667173306147
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,63,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,127,0.010794666906197866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,127,0.010501333822806677
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,255,0.011050666371981302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,255,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,511,0.011183999478816986
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,511,0.01102399950226148
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,1023,0.012986666212479273
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,1023,0.01097600037852923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,2047,0.013568000247081121
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,2047,0.012821332861979803
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,4095,0.02992533395687739
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,4095,0.015311999867359797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,16383,0.08270933230717976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,8191,0.031194667021433514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,8191,0.04799466828505198
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,16383,0.04826133449872335
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,1,0.010496000448862711
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,3,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,1,0.011087999989589056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,3,0.010928000013033548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,7,0.011253333340088526
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,15,0.01109333336353302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,7,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,15,0.010735999792814255
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,31,0.01102399950226148
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,31,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,63,0.010853332777818045
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,63,0.011029332876205444
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,127,0.011130666981140772
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,127,0.011120000233252844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,255,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,255,0.01071999967098236
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,511,0.010938666760921478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,511,0.011264000087976456
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,1023,0.013434667140245438
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,1023,0.013045333325862885
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,2047,0.02959466725587845
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,2047,0.015200000256299973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,4095,0.03011200080315272
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,4095,0.046581332882245384
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,8191,0.08140799899895985
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,8191,0.0481279989083608
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,16383,0.1504800021648407
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,1,0.010847999403874079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,1,0.010693332801262537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,3,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,16383,0.08322133123874664
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,3,0.009093333035707474
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,7,0.010725333044926325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,15,0.008901333436369896
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,7,0.010821333775917688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,15,0.009279999881982803
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,31,0.010762666662534079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,31,0.009119999905427298
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,63,0.010751999914646149
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,63,0.00960533320903778
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,127,0.009029333169261614
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,255,0.010938666760921478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,127,0.010842667271693548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,511,0.011029332876205444
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,511,0.011168000598748526
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,255,0.011029332876205444
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,1023,0.010832000523805618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,1023,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,2047,0.011237333218256632
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,4095,0.01116266722480456
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,2047,0.011551999797423681
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,4095,0.011050666371981302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,8191,0.015141333142916361
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,8191,0.015941333025693893
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,16383,0.019498666127522785
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,16383,0.019189332922299702
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,32767,0.02752533306678136
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,65535,0.03181866556406021
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,65535,0.03136533250411352
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,1,0.009312000125646591
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,32767,0.0277813325325648
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,3,0.00914666677514712
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,1,0.009514666472872099
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,3,0.010346666599313417
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,7,0.009663999701539675
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,7,0.008789333204428354
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,15,0.008912000184257826
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,15,0.010634666929642359
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,31,0.009418666362762451
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,31,0.009056000038981438
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,63,0.008912000184257826
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,63,0.010709332923094431
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,127,0.009583999713261923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,127,0.008805333326260248
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,255,0.010522666076819101
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,255,0.009173333023985228
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,511,0.010735999792814255
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,511,0.010842667271693548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,1023,0.010970667004585266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,1023,0.011050666371981302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,2047,0.010784000158309937
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,2047,0.01099733387430509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,4095,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,4095,0.011120000233252844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,8191,0.015034666905800501
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,16383,0.019253333409627277
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,16383,0.01741333305835724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,8191,0.015562667200962702
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,32767,0.021402666966120403
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,32767,0.02139200021823247
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,65535,0.023306667804718018
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,65535,0.023584000766277313
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,1,0.00921066664159298
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,1,0.010735999792814255
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,3,0.008837333569924036
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,3,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,7,0.009050666665037474
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,7,0.010618666807810465
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,15,0.008976000050703684
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,15,0.010640000303586325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,31,0.010687999427318573
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,31,0.009103999783595404
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,63,0.00891733355820179
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,63,0.010847999403874079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,127,0.01020800011853377
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,127,0.009423999736706415
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,255,0.008901333436369896
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,255,0.010656000425418219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,511,0.011136000355084738
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,511,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,1023,0.010869332899649939
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,1023,0.010559999694426855
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,2047,0.010992000500361124
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,2047,0.011114666859308878
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,4095,0.010847999403874079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,4095,0.011098666737476984
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,8191,0.015114666273196539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,8191,0.015013333410024643
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,16383,0.017093333105246227
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,16383,0.01749333366751671
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,32767,0.019189332922299702
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,32767,0.019199999670187633
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,65535,0.03716800113519033
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,65535,0.020015999674797058
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,1,0.008986666798591614
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,1,0.010672000547250112
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,3,0.009226666763424873
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,3,0.009205333267649015
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,7,0.010847999403874079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,7,0.00915733352303505
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,15,0.008933333059151968
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,15,0.010842667271693548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,31,0.009088000282645226
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,31,0.009093333035707474
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,63,0.010746666540702185
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,63,0.009589333087205887
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,127,0.008997333546479544
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,127,0.010869332899649939
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,255,0.009279999881982803
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,255,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,511,0.011264000087976456
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,511,0.011087999989589056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,1023,0.01110400011142095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,1023,0.01118933285276095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,2047,0.011034666250149408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,2047,0.011002667248249054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,4095,0.012975999464591345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,4095,0.01099733387430509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,8191,0.015040000279744467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,8191,0.01706133286158244
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,16383,0.019567999988794327
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,16383,0.019530666371186573
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,32767,0.0236160010099411
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,32767,0.02333866556485494
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,65535,0.025253333151340485
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,65535,0.02566933383544286
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,1,0.009029333169261614
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,3,0.009216000015536943
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,3,0.010666667173306147
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,1,0.011130666981140772
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,7,0.009775999933481216
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,7,0.008853333070874214
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,15,0.009359999870260557
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,31,0.009925333162148794
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,31,0.009119999905427298
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,15,0.011168000598748526
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,63,0.009397333487868309
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,63,0.00916800027092298
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,127,0.00921066664159298
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,255,0.009365333244204521
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,127,0.010874666273593903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,255,0.008890666688481966
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,511,0.010527999450763067
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,511,0.010666667173306147
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,1023,0.010725333044926325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,2047,0.010816000401973724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,1023,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,2047,0.011007999380429586
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,4095,0.010751999914646149
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,4095,0.010911999891201654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,8191,0.015311999867359797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,16383,0.017125333348910015
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,8191,0.01605333387851715
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,16383,0.0170666662355264
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,32767,0.02006400004029274
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,32767,0.01913600042462349
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,65535,0.020021333048741024
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,65535,0.03664000084002813
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,1,0.010778666784365972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,1,0.010954666882753372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,3,0.009029333169261614
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,3,0.0107893335322539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,7,0.0107893335322539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,7,0.009535999968647957
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,15,0.009519999846816063
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,15,0.011007999380429586
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,31,0.010645333677530289
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,31,0.009183999771873156
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,63,0.009525333220760027
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,63,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,127,0.008885333314538002
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,127,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,255,0.008863999818762144
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,255,0.010837333897749582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,511,0.0106133334338665
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,511,0.010847999403874079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,1023,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,2047,0.01098666712641716
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,1023,0.01091733326514562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,2047,0.010677333921194077
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,4095,0.013061333447694778
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,4095,0.013023999830087027
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,8191,0.014874666929244995
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,8191,0.015376000354687372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,16383,0.017045332739750545
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,16383,0.016762666404247284
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,32767,0.033370666205883026
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,32767,0.017738666385412216
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,65535,0.05158400038878123
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,65535,0.03477333237727483
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,1,0.010847999403874079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,1,0.011109333485364914
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,3,0.010847999403874079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,3,0.010698666175206503
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,7,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,7,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,15,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,15,0.011050666371981302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,31,0.010853332777818045
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,31,0.010640000303586325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,63,0.010853332777818045
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,63,0.01080000028014183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,127,0.010746666540702185
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,255,0.010784000158309937
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,127,0.011247999966144562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,255,0.011258666714032492
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,511,0.011301333705584208
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,511,0.012901333471139273
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,1023,0.0129120002190272
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,1023,0.012949333836634954
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,2047,0.014991999914248785
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,2047,0.014965333044528961
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,8191,0.04820266862710317
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,4095,0.029845332105954487
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,8191,0.03295466552178065
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,4095,0.01810666670401891
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,1,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,1,0.011034666250149408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,3,0.010656000425418219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,3,0.01071999967098236
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,7,0.010981333752473196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,7,0.010928000013033548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,15,0.01062400018175443
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,15,0.011007999380429586
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,31,0.01080000028014183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,31,0.01080000028014183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,63,0.010853332777818045
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,63,0.010762666662534079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,127,0.010944000134865442
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,127,0.011007999380429586
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,255,0.010853332777818045
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,255,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,511,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,511,0.01119999960064888
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,1023,0.013183999806642532
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,1023,0.012901333471139273
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,2047,0.029359998802344005
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,4095,0.047637333472569786
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,2047,0.015013333410024643
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,4095,0.0306986669699351
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,8191,0.08258666594823201
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,8191,0.04797866443792979
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,1,0.012954667210578918
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,1,0.011887999872366587
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,3,0.01116266722480456
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,3,0.011247999966144562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,7,0.012730666746695837
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,7,0.012122667084137598
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,15,0.012261333564917246
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,15,0.012789333860079447
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,31,0.01121066634853681
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,31,0.01249066616098086
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,63,0.012416000167528788
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,63,0.012757333616415659
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,127,0.011349332829316458
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,127,0.012768000364303589
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,255,0.012543999900420507
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,255,0.012149333953857422
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,511,0.01413333291808764
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,511,0.01339200014869372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,1023,0.015114666273196539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,1023,0.029466666281223297
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,2047,0.04783466458320618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,2047,0.03148266673088074
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,4095,0.08247466882069905
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,4095,0.04795733094215393
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,8191,0.15170133113861084
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,1,0.010885333021481832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,1,0.01099733387430509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,3,0.008885333314538002
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,3,0.010757333288590113
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,7,0.010672000547250112
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,8191,0.0846453309059143
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,7,0.010586666564146677
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,15,0.009893333539366722
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,15,0.010618666807810465
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,31,0.009066666786869368
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,31,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,63,0.009002666920423508
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,63,0.010709332923094431
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,127,0.010853332777818045
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,127,0.010762666662534079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,255,0.008933333059151968
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,255,0.010911999891201654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,511,0.010885333021481832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,511,0.010928000013033548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,1023,0.01121066634853681
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,1023,0.01109333336353302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,2047,0.01080000028014183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,2047,0.011120000233252844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,4095,0.011018666128317514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,4095,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,8191,0.01693333312869072
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,8191,0.016858667135238647
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,16383,0.01897066707412402
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,16383,0.019248000035683315
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,32767,0.021242665747801464
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,32767,0.019445333629846573
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,65535,0.03774400055408478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,65535,0.021840001145998638
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,1,0.009434666484594345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,3,0.008879999940594038
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,1,0.010741333166758219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,3,0.009205333267649015
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,7,0.00921066664159298
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,7,0.00979200005531311
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,15,0.00922133338948091
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,15,0.010608000059922537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,31,0.009637333452701569
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,31,0.009056000038981438
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,63,0.009093333035707474
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,63,0.010687999427318573
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,127,0.00892800030608972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,127,0.010586666564146677
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,255,0.009648000200589498
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,255,0.009226666763424873
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,511,0.010805333654085795
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,511,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,1023,0.010981333752473196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,1023,0.011071999867757162
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,2047,0.01110400011142095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,2047,0.010832000523805618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,4095,0.01303999995191892
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,4095,0.01312000056107839
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,8191,0.014943999548753103
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,16383,0.017103999853134155
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,16383,0.016986666868130367
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,8191,0.01613866661985715
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,32767,0.018415999909241993
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,32767,0.03338133295377096
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,65535,0.05212800204753876
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,1,0.008997333546479544
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,65535,0.03577066709597906
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,1,0.009952000031868616
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,3,0.010512000570694605
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,3,0.00921066664159298
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,7,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,15,0.010757333288590113
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,7,0.010319999729593595
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,15,0.00919999989370505
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,31,0.010250666489203772
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,31,0.010768000036478043
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,63,0.009093333035707474
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,63,0.010832000523805618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,127,0.010714666297038397
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,127,0.011071999867757162
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,255,0.008933333059151968
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,511,0.011071999867757162
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,511,0.011007999380429586
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,255,0.011274666835864386
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,1023,0.010805333654085795
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,1023,0.010784000158309937
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,2047,0.012890666723251343
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,2047,0.01328533391157786
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,4095,0.013232000172138214
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,4095,0.014831999937693277
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,8191,0.016389333953460056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,16383,0.03139200061559677
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,16383,0.01762666677435239
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,8191,0.015546667079130808
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,32767,0.04949333270390829
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,32767,0.03282133241494497
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,65535,0.08405333757400513
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,65535,0.051882664362589516
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,1,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,1,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,3,0.012608000387748083
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,3,0.011557333171367645
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,7,0.01091733326514562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,7,0.010944000134865442
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,15,0.011098666737476984
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,31,0.010821333775917688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,15,0.011359999577204386
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,31,0.010874666273593903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,63,0.01137599969903628
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,63,0.010794666906197866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,127,0.010944000134865442
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,127,0.011509332805871964
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,255,0.011071999867757162
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,255,0.011237333218256632
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,511,0.012730666746695837
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,1023,0.014864000181357065
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,511,0.011733333269755045
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,1023,0.016000000139077503
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,2047,0.029722665747006733
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,2047,0.017530667285124462
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,4095,0.04784533381462097
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,4095,0.03139200061559677
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,1,0.012981332838535309
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,1,0.011173332730929056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,3,0.01102399950226148
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,3,0.011450666934251785
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,7,0.011071999867757162
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,15,0.01098666712641716
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,7,0.013232000172138214
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,15,0.011338666081428528
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,31,0.012304000556468964
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,31,0.011039999624093374
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,63,0.011610666910807291
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,63,0.012341332932313284
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,127,0.013088000317414602
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,127,0.011215999722480774
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,255,0.012965332716703415
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,255,0.01163200040658315
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,511,0.01492799942692121
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,511,0.013189333180586496
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,1023,0.02979733298222224
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,1023,0.016373333831628162
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,2047,0.031386665999889374
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,2047,0.04842133323351542
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,4095,0.08492799599965413
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,4095,0.04976533353328705
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,1,0.01469333345691363
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,1,0.012997332960367203
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,3,0.013349333157142004
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,3,0.014922666052977243
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,7,0.014933332800865173
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,7,0.013306666165590286
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,15,0.015130666395028433
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,15,0.014901333798964819
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,31,0.012928000340859095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,31,0.014943999548753103
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,63,0.014890667051076889
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,63,0.014954666296641031
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,127,0.012928000340859095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,127,0.01471466695268949
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,255,0.013056000073750814
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,255,0.015050667027632395
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,511,0.02978666623433431
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,511,0.017210666090250015
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,2047,0.049973333875338234
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,2047,0.08330133557319641
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,1023,0.032645332316557564
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,1023,0.048570667703946434
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,4095,0.08592533071835835
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,4095,0.15430933237075806
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,1,0.012869333227475485
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,1,0.012890666723251343
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,3,0.012997332960367203
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,3,0.012917333592971167
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,7,0.013013333082199097
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,7,0.012714666624863943
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,15,0.013061333447694778
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,15,0.012885333349307379
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,31,0.01301866645614306
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,31,0.012965332716703415
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,63,0.012944000462690989
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,63,0.01292266696691513
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,127,0.012874666601419449
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,127,0.013023999830087027
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,255,0.012975999464591345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,255,0.012965332716703415
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,511,0.015247999380032221
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,511,0.01515199989080429
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,1023,0.031184000273545582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,1023,0.0170666662355264
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,1,0.014954666296641031
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,1,0.014826666563749313
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,3,0.014938666174809137
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,3,0.01301866645614306
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,7,0.014938666174809137
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,15,0.014922666052977243
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,15,0.013056000073750814
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,7,0.014943999548753103
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,31,0.014869333555301031
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,31,0.013237333546082178
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,63,0.014688000082969666
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,63,0.01504533365368843
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,127,0.014933332800865173
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,127,0.013248000293970108
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,255,0.01498666654030482
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,255,0.014842666685581207
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,511,0.03158933420976003
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,511,0.017221332838137943
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,1023,0.04826133449872335
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,1023,0.032127998769283295
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,1,0.019237333287795384
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,1,0.016997333616018295
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,3,0.018917333334684372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,3,0.017173333714405697
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,7,0.019071999937295914
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,7,0.01709866647919019
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,15,0.01913600042462349
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,15,0.017103999853134155
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,31,0.018778666853904724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,31,0.0170666662355264
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,63,0.018960000326236088
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,63,0.017466666797796886
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,127,0.0191040001809597
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,127,0.01714133347074191
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,255,0.017279999951521557
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,255,0.029552000264326733
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,511,0.04804266492525736
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,511,0.03145600110292435
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,1023,0.0820853312810262
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,1023,0.04969066878159841
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,1,0.015850666910409927
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,1,0.014949332922697067
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,3,0.016895999511082966
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,3,0.014949332922697067
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,7,0.015813333292802174
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,7,0.01488000030318896
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,15,0.016869333883126576
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,15,0.015173333386580149
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,31,0.015925332903862
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,31,0.014949332922697067
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,63,0.017173333714405697
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,63,0.014874666929244995
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,127,0.015749332805474598
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,127,0.014917333920796713
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,255,0.015967999895413715
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,255,0.015157333264748255
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,511,0.030799999833106995
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,511,0.019226666539907455
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,1,0.01933866615096728
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,1,0.017658667018016178
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,3,0.019109333554903667
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,3,0.017018667111794155
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,7,0.019333332777023315
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,15,0.01916266605257988
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,7,0.017557332913080852
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,15,0.017301333447297413
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,31,0.018885333091020584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,31,0.017301333447297413
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,63,0.01709866647919019
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,63,0.019434666881958645
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,127,0.01916266605257988
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,127,0.017045332739750545
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,255,0.029994666576385498
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,255,0.017573333034912746
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,1,0.029285334050655365
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,511,0.031343999008337654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,511,0.049098665515581764
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,1,0.027658666173617046
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,3,0.02939733366171519
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,3,0.027274665733178455
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,7,0.029711998999118805
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,7,0.027514666318893433
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,15,0.02939733366171519
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,15,0.02737066646416982
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,31,0.02958400050799052
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,31,0.027232001225153606
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,63,0.029461334149042766
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,63,0.027669332921504974
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,127,0.035455999275048576
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,127,0.02759466568628947
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,255,0.050288001696268715
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,1,0.010687999427318573
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,255,0.03533333291610082
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,1,0.011029332876205444
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,3,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,3,0.010794666906197866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,7,0.010832000523805618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,7,0.010666667173306147
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,15,0.011029332876205444
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,15,0.008938666433095932
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,31,0.010709332923094431
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,31,0.00915733352303505
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,63,0.009050666665037474
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,63,0.010245333115259806
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,511,0.085125337044398
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,127,0.010464000205198923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,127,0.010293333480755487
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,255,0.009232000137368837
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,511,0.011077333241701126
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,511,0.05274133384227753
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,511,0.010954666882753372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,1023,0.010911999891201654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,255,0.01116266722480456
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,2047,0.011087999989589056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,2047,0.01081066702802976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,4095,0.01523200049996376
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,4095,0.014943999548753103
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,8191,0.016965333372354507
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,1023,0.01118933285276095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,8191,0.016885332763195038
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,16383,0.01905599981546402
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,16383,0.017103999853134155
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,32767,0.03451200077931086
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,32767,0.02109866589307785
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,65535,0.05243733525276184
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,65535,0.035962666074434914
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,1,0.010741333166758219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,1,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,3,0.010501333822806677
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,3,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,7,0.010703999549150467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,7,0.010725333044926325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,15,0.009402666861812273
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,15,0.010768000036478043
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,31,0.01098666712641716
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,31,0.010319999729593595
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,63,0.010384000216921171
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,63,0.010832000523805618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,127,0.010757333288590113
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,127,0.009626666704813639
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,255,0.009535999968647957
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,255,0.01098666712641716
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,511,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,511,0.010821333775917688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,1023,0.011007999380429586
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,1023,0.011194666226704916
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,2047,0.012826666235923767
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,2047,0.012938667088747025
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,4095,0.01505600040157636
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,8191,0.015024000157912573
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,4095,0.013418667018413544
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,8191,0.015247999380032221
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,16383,0.03130666663249334
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,16383,0.01775466650724411
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,32767,0.050197333097457886
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,32767,0.032842665910720825
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,65535,0.05198933184146881
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,65535,0.0848533312479655
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,1,0.00914666677514712
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,1,0.01062400018175443
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,3,0.01091733326514562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,3,0.01080000028014183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,7,0.01101333275437355
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,7,0.01033599985142549
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,15,0.010362666721145311
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,15,0.01062400018175443
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,31,0.010746666540702185
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,31,0.01250133290886879
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,63,0.010821333775917688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,63,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,127,0.010794666906197866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,255,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,127,0.010202666744589806
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,255,0.010741333166758219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,511,0.010725333044926325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,511,0.010928000013033548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,1023,0.011109333485364914
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,1023,0.011136000355084738
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,2047,0.012890666723251343
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,2047,0.011183999478816986
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,4095,0.014943999548753103
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,8191,0.029466666281223297
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,4095,0.013440000514189402
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,8191,0.015082667271296183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,16383,0.030799999833106995
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,16383,0.04765866696834564
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,32767,0.048063998421033226
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,32767,0.08157866696516673
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,65535,0.1504853367805481
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,65535,0.08542933066685994
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,1,0.02146666745344798
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,3,0.023077333966890972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,1,0.019189332922299702
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,3,0.019551999866962433
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,7,0.021488000949223835
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,7,0.019359999646743137
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,15,0.023029332359631855
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,15,0.019445333629846573
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,31,0.021509334444999695
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,31,0.019205333044131596
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,63,0.023157333334287006
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,63,0.019546666493018467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,127,0.02164799968401591
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,127,0.01922133316596349
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,255,0.031397332747777305
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,255,0.019440000255902607
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,1,0.029274667302767437
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,1,0.02722666660944621
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,3,0.02938133229811986
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,3,0.02737066646416982
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,7,0.029359998802344005
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,7,0.027450665831565857
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,15,0.029343999922275543
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,15,0.02720000098148982
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,31,0.02959999938805898
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,31,0.027445333699385326
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,63,0.02935466667016347
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,63,0.027488000690937042
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,127,0.035461333890755974
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,127,0.027471999327341717
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,255,0.052000001072883606
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,255,0.03578133384386698
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,1,0.053861334919929504
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,1,0.04557866851488749
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,3,0.05363733569780985
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,3,0.045552000403404236
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,7,0.054154664278030396
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,7,0.045834665497144066
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,15,0.05403733253479004
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,15,0.04576533536116282
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,31,0.05410666763782501
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,63,0.05393599967161814
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,31,0.04586666822433472
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,63,0.04540266593297323
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,127,0.06066666543483734
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,127,0.0514933317899704
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,255,0.06010666489601135
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,255,0.09115200241406758
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,1,0.0365280012289683
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,1,0.031370667119820915
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,3,0.035386666655540466
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,3,0.0315786674618721
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,7,0.035717333356539406
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,7,0.031343999008337654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,15,0.035904000202814736
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,15,0.031397332747777305
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,31,0.035589332381884255
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,31,0.03145600110292435
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,63,0.03552533437808355
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,63,0.03139200061559677
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,127,0.03948266555865606
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,127,0.031301334500312805
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,1,0.052527998884518944
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,1,0.0440586656332016
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,3,0.04375466704368591
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,3,0.05347733199596405
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,7,0.05246399839719137
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,7,0.04573333263397217
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,15,0.05260799825191498
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,15,0.04483200112978617
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,31,0.044213334719340004
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,31,0.052383999029795326
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,63,0.05227733155091604
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,63,0.043824002146720886
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,127,0.058448001742362976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,127,0.04943466683228811
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,1,0.0951039989789327
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,1,0.08876799543698628
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,3,0.08876799543698628
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,3,0.09947733084360759
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,7,0.08880533774693807
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,7,0.09675733248392741
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,15,0.09790399670600891
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,15,0.08896533648173015
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,31,0.09749866525332133
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,31,0.08693866928418477
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,63,0.09750399986902873
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,1,0.011007999380429586
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,1,0.010239999741315842
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,3,0.009258666386206945
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,3,0.010250666489203772
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,7,0.008901333436369896
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,7,0.01080000028014183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,63,0.08891200025876363
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,15,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,31,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,15,0.010746666540702185
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,31,0.010741333166758219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,63,0.010762666662534079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,63,0.010751999914646149
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,127,0.010821333775917688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,127,0.010725333044926325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,255,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,255,0.010885333021481832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,511,0.010928000013033548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,511,0.01121066634853681
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,1023,0.010911999891201654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,1023,0.011328000575304031
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,2047,0.01313599944114685
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,127,0.10795199871063232
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,2047,0.01314666618903478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,4095,0.015184000134468079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,4095,0.01522133375207583
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,8191,0.017055999487638474
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,16383,0.020154666155576706
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,16383,0.033514666060606636
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,8191,0.01709866647919019
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,32767,0.05042133231957754
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,1,0.010677333921194077
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,32767,0.03517866631348928
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,1,0.010911999891201654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,3,0.010389333590865135
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,3,0.01073066641887029
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,7,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,7,0.010842667271693548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,15,0.010421333213647207
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,15,0.009066666786869368
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,31,0.01062400018175443
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,31,0.01101333275437355
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,63,0.00914666677514712
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,63,0.010741333166758219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,127,0.010581333190202713
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,127,0.00927466650803884
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,255,0.01073066641887029
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,255,0.010768000036478043
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,511,0.010842667271693548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,511,0.010746666540702185
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,1023,0.01129066695769628
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,1023,0.01101333275437355
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,2047,0.01267733300725619
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,2047,0.01312000056107839
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,4095,0.014954666296641031
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,4095,0.01328533391157786
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,8191,0.02994133283694585
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,8191,0.01701333373785019
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,127,0.09084266424179077
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,16383,0.048122664292653404
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,16383,0.03172266731659571
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,32767,0.08341333270072937
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,32767,0.04971200227737427
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,1,0.010565333068370819
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,3,0.01098666712641716
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,1,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,3,0.010608000059922537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,7,0.010842667271693548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,7,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,15,0.010751999914646149
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,31,0.010570666442314783
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,31,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,15,0.011066666493813196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,63,0.010634666929642359
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,63,0.010842667271693548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,127,0.010757333288590113
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,127,0.011066666493813196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,255,0.010778666784365972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,255,0.010751999914646149
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,511,0.011055999745925268
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,1023,0.012858666479587555
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,511,0.010714666297038397
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,2047,0.012954667210578918
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,1023,0.012885333349307379
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,2047,0.013237333546082178
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,4095,0.02922133356332779
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,4095,0.015024000157912573
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,8191,0.04799466828505198
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,8191,0.031210665901501972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,16383,0.08135466774304707
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,16383,0.04786133269468943
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,32767,0.08393067121505737
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,1,0.010597333312034607
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,32767,0.1508853336175283
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,1,0.009125333279371262
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,3,0.009541333342591921
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,3,0.010911999891201654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,7,0.009237333511312803
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,7,0.009925333162148794
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,15,0.010037333394090334
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,15,0.01097600037852923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,31,0.009018666421373686
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,31,0.010725333044926325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,63,0.010058666889866194
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,63,0.009290666629870733
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,127,0.008965333302815756
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,127,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,255,0.010693332801262537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,255,0.009893333539366722
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,511,0.010816000401973724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,511,0.011178666104873022
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,1023,0.01116266722480456
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,1023,0.011061333119869232
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,2047,0.013034666577974955
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,2047,0.012949333836634954
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,4095,0.014997333288192749
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,4095,0.013093333691358566
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,8191,0.0306986669699351
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,8191,0.016821333517630894
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,16383,0.048325334986050926
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,16383,0.03137599925200144
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,32767,0.08454933762550354
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,1,0.010768000036478043
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,32767,0.05000533163547516
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,1,0.010981333752473196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,3,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,3,0.01080000028014183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,7,0.010874666273593903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,7,0.01080000028014183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,15,0.01098666712641716
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,15,0.011029332876205444
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,31,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,31,0.010741333166758219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,63,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,63,0.01108266661564509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,127,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,127,0.010842667271693548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,255,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,255,0.01062400018175443
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,511,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,511,0.01145600030819575
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,1023,0.012954667210578918
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,1023,0.013072000195582708
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,2047,0.013690666606028875
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,4095,0.02957333376010259
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,2047,0.01328533391157786
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,4095,0.016421332955360413
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,8191,0.04807466765244802
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,8191,0.030618667602539062
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,16383,0.048991998036702476
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,16383,0.08328533172607422
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,32767,0.15319466590881348
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,1,0.010954666882753372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,32767,0.08500799536705017
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,1,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,3,0.011055999745925268
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,3,0.010794666906197866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,7,0.010784000158309937
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,7,0.011136000355084738
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,15,0.010746666540702185
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,15,0.010970667004585266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,31,0.011109333485364914
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,31,0.011434666812419891
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,63,0.010693332801262537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,63,0.011152000476916632
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,127,0.011242666592200598
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,127,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,255,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,255,0.01101333275437355
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,511,0.011157333850860596
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,1023,0.013173333058754602
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,511,0.01137599969903628
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,1023,0.013173333058754602
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,2047,0.029189333319664
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,2047,0.015205333630243937
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,4095,0.046351999044418335
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,8191,0.08070933322111766
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,4095,0.029839999973773956
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,8191,0.04790933430194855
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,16383,0.08297599852085114
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,16383,0.15074666341145834
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,32767,0.287498672803243
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,32767,0.15386666854222616
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,1,0.01163200040658315
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,1,0.011440000186363855
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,3,0.011039999624093374
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,3,0.012613333761692047
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,7,0.01099733387430509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,7,0.011498666057984034
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,15,0.011285333583752314
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,15,0.012058666596810022
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,31,0.011178666104873022
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,31,0.011157333850860596
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,63,0.011125333607196808
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,63,0.01145600030819575
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,127,0.012170666207869848
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,127,0.01109333336353302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,255,0.012821332861979803
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,255,0.011242666592200598
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,511,0.014725333700577417
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,1023,0.029605334003766377
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,511,0.013002666334311167
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,1023,0.01533866673707962
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,2047,0.04781333108743032
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,2047,0.031343999008337654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,4095,0.048298666874567665
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,4095,0.08212799827257793
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,8191,0.15053332845369974
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,8191,0.08359466989835103
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,16383,0.15169066190719604
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,16383,0.29126399755477905
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,1,0.008965333302815756
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,1,0.010682666053374609
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,32767,0.5693973302841187
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,32767,0.29229867458343506
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,3,0.009077333534757296
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,3,0.010794666906197866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,15,0.00901333304742972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,7,0.009103999783595404
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,15,0.010794666906197866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,7,0.010581333190202713
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,31,0.00902399979531765
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,31,0.009072000160813332
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,63,0.010757333288590113
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,63,0.00903466654320558
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,127,0.008965333302815756
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,127,0.009183999771873156
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,255,0.010538666198650995
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,255,0.010746666540702185
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,511,0.011050666371981302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,1023,0.010832000523805618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,511,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,1023,0.010757333288590113
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,2047,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,2047,0.010634666929642359
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,4095,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,4095,0.011141333729028702
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,8191,0.014959999670584997
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,16383,0.018992000569899876
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,8191,0.014789332946141561
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,32767,0.02588266630967458
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,16383,0.01887999971707662
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,32767,0.02717866748571396
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,65535,0.030282666285832722
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,65535,0.029274667302767437
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,131071,0.03201599915822347
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,131071,0.03169599920511246
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,1,0.00922133338948091
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,1,0.009632000078757605
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,3,0.009189333145817121
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,7,0.00919999989370505
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,3,0.009503999724984169
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,7,0.00890666681031386
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,15,0.009242666885256767
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,15,0.010821333775917688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,31,0.00903466654320558
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,31,0.010645333677530289
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,63,0.01121066634853681
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,63,0.009205333267649015
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,127,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,127,0.011141333729028702
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,255,0.008826666822036108
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,255,0.010698666175206503
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,511,0.011120000233252844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,511,0.010672000547250112
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,1023,0.011039999624093374
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,1023,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,2047,0.010874666273593903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,2047,0.01108266661564509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,4095,0.011338666081428528
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,4095,0.010911999891201654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,8191,0.014959999670584997
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,16383,0.01913600042462349
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,8191,0.015295999745527903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,16383,0.019194666296243668
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,32767,0.0222080002228419
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,32767,0.02183466653029124
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,65535,0.023520000278949738
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,65535,0.023120000958442688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,131071,0.025125332176685333
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,131071,0.04195199906826019
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,1,0.009178666397929192
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,1,0.010762666662534079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,3,0.010191999996701876
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,3,0.009530666594703993
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,7,0.010602666685978571
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,15,0.00895999992887179
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,15,0.010149333626031876
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,7,0.009679999823371569
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,31,0.010645333677530289
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,31,0.009098666409651438
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,63,0.009829333052039146
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,63,0.009205333267649015
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,127,0.010629333555698395
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,127,0.008853333070874214
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,255,0.011178666104873022
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,255,0.009653333574533463
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,511,0.010794666906197866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,511,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,1023,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,1023,0.010874666273593903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,2047,0.010768000036478043
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,2047,0.010682666053374609
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,8191,0.015381333728631338
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,4095,0.011007999380429586
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,8191,0.01534933348496755
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,4095,0.011370666325092316
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,16383,0.016895999511082966
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,16383,0.017429333180189133
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,32767,0.02109866589307785
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,32767,0.019402666638294857
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,65535,0.035786665976047516
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,65535,0.021151999632517498
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,131071,0.05421866476535797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,131071,0.038058665891488395
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,1,0.00895999992887179
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,1,0.010869332899649939
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,3,0.010015999898314476
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,3,0.009242666885256767
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,7,0.010197333370645842
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,7,0.010533332824707031
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,15,0.009317333499590555
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,15,0.010213333492477735
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,31,0.009349333122372627
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,31,0.010656000425418219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,63,0.010837333897749582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,63,0.009941333283980688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,127,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,127,0.01073066641887029
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,255,0.01055466632048289
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,255,0.01080000028014183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,511,0.01081066702802976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,1023,0.011152000476916632
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,511,0.0107893335322539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,1023,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,2047,0.010869332899649939
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,2047,0.011152000476916632
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,4095,0.013770667215188345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,4095,0.013258667041858038
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,8191,0.015061333775520325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,8191,0.014922666052977243
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,16383,0.017530667285124462
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,16383,0.0169813334941864
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,32767,0.017312000195185345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,32767,0.033200000723203026
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,65535,0.05036266644795736
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,65535,0.03478399912516276
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,1,0.00915733352303505
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,131071,0.05177066723505656
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,131071,0.08458667000134786
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,1,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,3,0.009472000102202097
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,3,0.009008000294367472
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,7,0.01091733326514562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,7,0.009632000078757605
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,15,0.008976000050703684
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,15,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,31,0.009322666873534521
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,31,0.009002666920423508
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,63,0.009999999776482582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,63,0.010778666784365972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,127,0.009365333244204521
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,127,0.009573333586255709
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,255,0.010970667004585266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,255,0.008976000050703684
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,511,0.010778666784365972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,511,0.01118933285276095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,1023,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,1023,0.01097600037852923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,2047,0.0107893335322539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,2047,0.01109333336353302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,4095,0.011178666104873022
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,4095,0.011381333072980246
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,8191,0.014975999792416891
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,8191,0.0169813334941864
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,16383,0.01941866676012675
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,16383,0.019194666296243668
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,32767,0.023061332603295643
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,32767,0.021231998999913532
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,65535,0.022970666488011677
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,65535,0.024085332949956257
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,131071,0.03924266745646795
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,131071,0.02550933261712392
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,1,0.00979200005531311
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,1,0.009301333377758661
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,3,0.009269333134094873
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,3,0.010543999572594961
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,7,0.008922666932145754
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,7,0.009685333197315535
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,15,0.009541333342591921
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,15,0.008832000195980072
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,31,0.010618666807810465
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,31,0.01073066641887029
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,63,0.00891733355820179
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,127,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,63,0.009621333330869675
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,127,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,255,0.009018666421373686
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,255,0.010656000425418219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,511,0.011114666859308878
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,511,0.010666667173306147
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,1023,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,1023,0.00996800015370051
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,2047,0.011050666371981302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,4095,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,2047,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,4095,0.011173332730929056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,8191,0.014997333288192749
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,16383,0.017429333180189133
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,8191,0.015493333339691162
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,16383,0.017701332767804463
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,32767,0.01932799940307935
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,32767,0.019317333896954853
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,65535,0.037087999284267426
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,65535,0.021301334102948506
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,131071,0.03751466671625773
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,131071,0.054245332876841225
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,1,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,1,0.009866666669646898
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,3,0.010751999914646149
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,3,0.009626666704813639
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,7,0.010970667004585266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,7,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,15,0.009866666669646898
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,15,0.0107893335322539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,31,0.00879466657837232
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,31,0.011312000453472137
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,63,0.010490667074918747
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,63,0.009621333330869675
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,127,0.008885333314538002
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,127,0.010741333166758219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,255,0.01097600037852923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,255,0.009839999799927076
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,511,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,511,0.01109333336353302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,1023,0.010794666906197866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,1023,0.011146667102972666
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,2047,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,2047,0.010874666273593903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,4095,0.013237333546082178
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,4095,0.013045333325862885
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,8191,0.015040000279744467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,8191,0.01552533358335495
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,16383,0.015504000087579092
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,16383,0.01695999999841054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,32767,0.033413333197434746
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,32767,0.018911999960740406
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,65535,0.034058667719364166
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,65535,0.04977599779764811
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,131071,0.08480000495910645
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,131071,0.05386666456858317
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,1,0.010837333897749582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,1,0.009173333023985228
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,3,0.010741333166758219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,3,0.010965333630641302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,7,0.010703999549150467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,7,0.008879999940594038
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,15,0.01097600037852923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,15,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,31,0.009712000067035357
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,31,0.010778666784365972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,63,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,63,0.011029332876205444
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,127,0.01091733326514562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,127,0.011034666250149408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,255,0.009530666594703993
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,255,0.010559999694426855
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,511,0.011055999745925268
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,511,0.01097600037852923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,1023,0.011194666226704916
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,1023,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,2047,0.01302933320403099
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,2047,0.012714666624863943
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,4095,0.014741333822409311
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,4095,0.015072000523408255
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,8191,0.016719999412695568
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,8191,0.014991999914248785
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,16383,0.03146133323510488
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,16383,0.01806933308641116
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,32767,0.049973333875338234
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,32767,0.032442666590213776
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,65535,0.08303999900817871
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,65535,0.0506879985332489
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,131071,0.08435199658075969
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,131071,0.15100266536076865
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,1,0.010757333288590113
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,3,0.009461333354314169
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,3,0.01097600037852923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,1,0.011226666470368704
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,7,0.010746666540702185
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,7,0.010656000425418219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,15,0.011141333729028702
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,15,0.01110400011142095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,31,0.010682666053374609
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,31,0.0107893335322539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,63,0.010762666662534079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,63,0.011061333119869232
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,127,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,127,0.010757333288590113
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,255,0.010693332801262537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,255,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,511,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,511,0.011039999624093374
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,1023,0.01268799975514412
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,1023,0.010938666760921478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,2047,0.013493333011865616
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,2047,0.013487999637921652
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,4095,0.030005333324273426
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,8191,0.04797866443792979
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,4095,0.015082667271296183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,8191,0.03134933362404505
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,16383,0.08422399560610454
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,16383,0.049551998575528465
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,1,0.010981333752473196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,1,0.011146667102972666
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,3,0.010650667051474253
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,3,0.01091733326514562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,7,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,7,0.010549332946538925
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,15,0.011077333241701126
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,15,0.01071999967098236
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,31,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,31,0.010992000500361124
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,63,0.010869332899649939
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,63,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,127,0.010816000401973724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,127,0.011173332730929056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,255,0.010816000401973724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,255,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,511,0.011018666128317514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,511,0.011247999966144562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,1023,0.013072000195582708
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,1023,0.013408000270525614
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,2047,0.02939733366171519
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,4095,0.047151997685432434
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,4095,0.03125333289305369
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,2047,0.015098666151364645
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,8191,0.08190933366616567
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,8191,0.04784533381462097
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,16383,0.15316800276438394
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,16383,0.08313600222269694
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,1,0.011952000359694162
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,1,0.011424000064531961
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,3,0.0129120002190272
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,3,0.011328000575304031
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,7,0.01116266722480456
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,7,0.011503999431928
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,15,0.012842666357755661
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,15,0.011066666493813196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,31,0.010874666273593903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,31,0.010970667004585266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,63,0.013104000439246496
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,63,0.01119999960064888
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,127,0.01267733300725619
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,127,0.012863999853531519
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,255,0.01121066634853681
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,255,0.011221333096424738
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,511,0.015061333775520325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,511,0.01310933381319046
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,1023,0.029232000311215717
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,2047,0.047637333472569786
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,1023,0.01700266698996226
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,2047,0.03062933435042699
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,4095,0.08212799827257793
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,4095,0.04866133133570353
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,8191,0.1511306663354238
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,1,0.014896000425020853
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,16383,0.290666659673055
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,8191,0.08519466718037923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,16383,0.15777599811553955
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,1,0.013269333789745966
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,3,0.01498666654030482
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,7,0.01516266663869222
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,3,0.01312000056107839
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,7,0.013023999830087027
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,15,0.014917333920796713
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,15,0.01321600005030632
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,31,0.014943999548753103
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,31,0.013264000415802002
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,63,0.01482133318980535
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,63,0.013151999562978745
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,127,0.012986666212479273
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,127,0.015008000036080679
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,255,0.01312000056107839
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,255,0.014815999815861383
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,511,0.02921066681543986
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,511,0.017504000415404636
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,1023,0.030640001098314922
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,1023,0.04822400212287903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,2047,0.0827893316745758
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,2047,0.0498933345079422
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,4095,0.0845973292986552
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,4095,0.1527786652247111
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,8191,0.2942933241526286
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,8191,0.1548533340295156
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,1,0.008858666444818178
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,3,0.009109333157539368
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,1,0.010768000036478043
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,16383,0.29516265789667767
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,16383,0.5776053269704183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,3,0.009141333401203156
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,7,0.00956266683836778
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,7,0.009232000137368837
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,15,0.008853333070874214
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,15,0.010591999938090643
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,31,0.009535999968647957
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,31,0.00892800030608972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,63,0.009466666728258133
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,63,0.010634666929642359
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,127,0.00997866690158844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,127,0.010170666500926018
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,255,0.008912000184257826
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,255,0.010677333921194077
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,511,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,511,0.010784000158309937
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,1023,0.010954666882753372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,1023,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,2047,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,2047,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,4095,0.010938666760921478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,4095,0.011066666493813196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,8191,0.015002666662136713
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,16383,0.01710933322707812
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,8191,0.01589866727590561
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,16383,0.017008000363906223
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,32767,0.019386666516462963
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,32767,0.01907733331123988
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,65535,0.03552533437808355
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,65535,0.021327999730904896
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,131071,0.053029333551724754
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,131071,0.035605333745479584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,1,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,1,0.010693332801262537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,3,0.009136000027259191
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,3,0.009045333291093508
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,7,0.010682666053374609
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,7,0.010656000425418219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,15,0.009109333157539368
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,15,0.009008000294367472
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,31,0.010832000523805618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,31,0.010778666784365972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,63,0.009658666948477427
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,63,0.009088000282645226
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,127,0.010698666175206503
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,127,0.01071999967098236
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,255,0.009381333366036415
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,255,0.009093333035707474
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,511,0.01091733326514562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,511,0.010954666882753372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,1023,0.01009599988659223
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,1023,0.010746666540702185
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,2047,0.011247999966144562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,2047,0.01101333275437355
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,4095,0.012975999464591345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,4095,0.013514666507641474
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,8191,0.015087999403476715
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,8191,0.014970666418472925
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,16383,0.01748266691962878
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,16383,0.01711999997496605
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,32767,0.03313066562016805
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,32767,0.018906666586796444
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,65535,0.05213333169619242
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,65535,0.034517332911491394
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,131071,0.08721066514650981
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,131071,0.05366399884223938
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,1,0.009232000137368837
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,1,0.010735999792814255
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,3,0.010197333370645842
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,3,0.010768000036478043
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,7,0.010608000059922537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,7,0.00960533320903778
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,15,0.009093333035707474
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,15,0.010645333677530289
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,31,0.00972800018886725
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,31,0.010751999914646149
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,63,0.008922666932145754
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,63,0.010586666564146677
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,127,0.008896000062425932
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,127,0.00915733352303505
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,255,0.009525333220760027
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,255,0.009109333157539368
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,511,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,511,0.010842667271693548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,1023,0.010762666662534079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,2047,0.012879999975363413
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,1023,0.01129066695769628
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,2047,0.012879999975363413
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,4095,0.013082666943470636
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,4095,0.013093333691358566
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,8191,0.01532799998919169
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,8191,0.016629333297411602
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,16383,0.03126399964094162
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,16383,0.01706133286158244
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,32767,0.04953599969546
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,32767,0.03339199970165888
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,65535,0.05065066615740458
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,65535,0.08284799754619598
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,1,0.00927466650803884
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,1,0.009039999917149544
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,131071,0.15239466230074564
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,131071,0.08565333485603333
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,3,0.009098666409651438
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,3,0.010437333335479101
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,7,0.01101333275437355
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,7,0.010677333921194077
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,15,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,15,0.010351999973257383
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,31,0.010735999792814255
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,31,0.010602666685978571
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,63,0.011861333002646765
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,63,0.010928000013033548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,127,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,127,0.010693332801262537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,255,0.010816000401973724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,255,0.010501333822806677
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,511,0.011183999478816986
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,511,0.011077333241701126
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,1023,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,1023,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,2047,0.012954667210578918
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,2047,0.01268799975514412
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,4095,0.013007999708255133
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,4095,0.013669333110253016
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,8191,0.029535998900731403
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,8191,0.01693333312869072
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,16383,0.04611733555793762
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,16383,0.030026666820049286
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,32767,0.08092266817887624
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,32767,0.04842666784922282
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,65535,0.08189866443475087
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,65535,0.14889066418011984
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,131071,0.28710933526357013
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,131071,0.15121066570281982
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,1,0.010778666784365972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,1,0.01098666712641716
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,3,0.01116266722480456
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,3,0.01166933278242747
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,7,0.010928000013033548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,7,0.010954666882753372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,15,0.011066666493813196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,15,0.011146667102972666
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,31,0.010853332777818045
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,31,0.011055999745925268
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,63,0.011109333485364914
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,63,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,127,0.010885333021481832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,127,0.01239466667175293
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,255,0.011055999745925268
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,255,0.010714666297038397
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,511,0.01097600037852923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,511,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,1023,0.013317332913478216
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,1023,0.013066666821638743
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,2047,0.02935466667016347
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,2047,0.015205333630243937
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,4095,0.04809600114822388
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,4095,0.031157332162062328
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,8191,0.08348266283671062
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,8191,0.04794133206208547
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,1,0.011168000598748526
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,1,0.012639999389648438
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,3,0.011776000261306763
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,7,0.011978667229413986
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,7,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,3,0.011029332876205444
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,15,0.012175999581813812
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,15,0.011152000476916632
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,31,0.011157333850860596
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,31,0.011482667177915573
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,63,0.011861333002646765
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,63,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,127,0.011168000598748526
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,127,0.012928000340859095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,255,0.012879999975363413
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,255,0.01091733326514562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,511,0.013712000101804733
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,511,0.013232000172138214
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,1023,0.030479999879995983
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,1023,0.015013333410024643
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,2047,0.04816000163555145
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,4095,0.08454400300979614
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,2047,0.03156800071398417
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,4095,0.0488373339176178
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,8191,0.15481600165367126
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,8191,0.08678399523099263
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,1,0.013045333325862885
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,1,0.015130666395028433
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,3,0.014709333578745524
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,3,0.014959999670584997
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,7,0.014080000420411428
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,7,0.013280000537633896
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,15,0.014906667172908783
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,15,0.014815999815861383
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,31,0.015061333775520325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,31,0.01314666618903478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,63,0.014954666296641031
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,63,0.013242666920026144
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,127,0.01504533365368843
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,127,0.013248000293970108
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,255,0.014954666296641031
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,255,0.013264000415802002
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,511,0.029765332738558452
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,511,0.017173333714405697
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,1023,0.04801600178082784
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,1023,0.03186666717131933
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,2047,0.08373866478602092
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,2047,0.05027199784914652
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,4095,0.15441067020098367
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,4095,0.08660266796747844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,8191,0.2958293358484904
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,8191,0.15741866827011108
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,1,0.019152000546455383
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,1,0.017530667285124462
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,3,0.018960000326236088
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,3,0.017279999951521557
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,7,0.01918399954835574
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,7,0.017370666066805523
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,15,0.019141333798567455
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,15,0.01746133342385292
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,31,0.018863999595244724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,31,0.017594666530688603
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,63,0.019146667172511418
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,63,0.017642666896184284
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,127,0.01940800001223882
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,127,0.017530667285124462
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,255,0.02926933268706004
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,255,0.01704000060757001
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,511,0.031301334500312805
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,511,0.04684799909591675
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,1023,0.08036266764005025
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,1023,0.04914666712284088
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,4095,0.285589337348938
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,2047,0.08286400139331818
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,2047,0.1500746707121531
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,4095,0.15201600392659506
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,8191,0.5589386622111002
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,8191,0.29121599594751996
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,1,0.01292266696691513
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,1,0.012736000120639801
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,3,0.011146667102972666
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,3,0.011221333096424738
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,7,0.01293333371480306
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,7,0.012837332983811697
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,15,0.013023999830087027
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,15,0.011178666104873022
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,31,0.012821332861979803
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,31,0.012938667088747025
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,63,0.012853333105643591
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,63,0.011525332927703857
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,127,0.012815999488035837
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,127,0.012970666090647379
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,255,0.01268799975514412
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,255,0.011690666278203329
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,511,0.015061333775520325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,511,0.012938667088747025
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,1023,0.030506665507952373
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,1023,0.016271999726692837
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,2047,0.03139200061559677
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,2047,0.04789866507053375
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,1,0.014890667051076889
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,1,0.012991999586423239
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,3,0.014970666418472925
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,7,0.015034666905800501
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,3,0.013210666676362356
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,7,0.01293333371480306
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,15,0.014266667266686758
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,15,0.015013333410024643
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,31,0.015370666980743408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,31,0.013130666067202887
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,63,0.014991999914248785
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,63,0.014842666685581207
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,127,0.015418666104475657
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,127,0.013082666943470636
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,255,0.01492799942692121
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,255,0.013349333157142004
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,511,0.030789333085219067
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,511,0.017103999853134155
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,1023,0.04924266537030538
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,1023,0.03196266790231069
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,2047,0.08449600140253703
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,2047,0.049914668003718056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,1,0.01913600042462349
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,1,0.017290666699409485
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,7,0.0189280000825723
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,7,0.01711999997496605
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,3,0.016976000120242436
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,3,0.01877333347996076
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,15,0.019215999792019527
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,15,0.017429333180189133
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,31,0.019088000059127808
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,31,0.017514667163292568
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,63,0.01911466692884763
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,63,0.01743999992807706
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,127,0.01894933357834816
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,127,0.017360000560681026
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,255,0.02940266579389572
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,255,0.017322666943073273
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,511,0.04772266745567322
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,511,0.03156266609827677
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,1023,0.08189866443475087
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,2047,0.1499626636505127
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,1023,0.04910400013128916
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,2047,0.08509332935015361
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,1,0.02754133443037669
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,1,0.02534399926662445
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,3,0.027653334041436512
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,3,0.02552533398071925
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,7,0.027237333357334137
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,7,0.025274666647116344
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,15,0.028117333849271137
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,15,0.02571733295917511
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,31,0.02756800005833308
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,31,0.025301332275072735
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,63,0.027615999182065327
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,63,0.02531733363866806
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,127,0.0323786661028862
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,127,0.025418666501839954
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,255,0.04799999793370565
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,255,0.032314665615558624
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,511,0.0814879983663559
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,511,0.04980266590913137
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,1023,0.1504800021648407
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,1023,0.08353599905967712
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,2047,0.28728000322977704
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,2047,0.15266133348147073
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,1,0.014848000059525171
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,1,0.014922666052977243
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,3,0.014917333920796713
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,3,0.013173333058754602
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,7,0.015173333386580149
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,7,0.014912000546852747
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,15,0.014725333700577417
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,15,0.013178666432698568
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,31,0.014981333166360855
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,31,0.01492799942692121
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,63,0.015013333410024643
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,63,0.013013333082199097
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,127,0.014869333555301031
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,127,0.015034666905800501
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,255,0.014970666418472925
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,255,0.013045333325862885
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,511,0.030293333033720653
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,511,0.01738133281469345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,1023,0.049178664882977806
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,1023,0.03183466692765554
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,1,0.019141333798567455
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,1,0.017397332936525345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,3,0.01912533367673556
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,3,0.01704000060757001
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,7,0.018965333700180054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,7,0.017338667064905167
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,15,0.019120000302791595
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,15,0.017429333180189133
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,31,0.01710933322707812
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,31,0.019274666905403137
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,63,0.019248000035683315
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,63,0.01749333366751671
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,127,0.01905599981546402
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,127,0.017610666652520496
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,255,0.01739199956258138
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,255,0.02994133283694585
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,511,0.047930667797724404
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,511,0.031311998764673867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,1023,0.08294400076071422
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,1,0.029365333418051403
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,1023,0.04967466493447622
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,1,0.027215999861558277
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,3,0.027221334477265675
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,3,0.029274667302767437
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,7,0.027456000447273254
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,7,0.029461334149042766
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,15,0.029264000554879505
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,15,0.02757333219051361
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,31,0.027376001079877216
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,31,0.02976000060637792
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,63,0.02934933453798294
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,63,0.027562665442625683
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,127,0.03515200068553289
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,127,0.027274665733178455
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,255,0.05052266518274943
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,255,0.03538133452335993
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,511,0.08458133538564046
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,511,0.052111998200416565
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,1023,0.15308266878128052
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,1,0.049770668148994446
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,1023,0.08796800176302592
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,1,0.04195733368396759
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,3,0.041696002086003624
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,3,0.05012799799442291
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,7,0.04200000067551931
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,7,0.05008533100287119
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,15,0.049733335773150124
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,15,0.04176533222198486
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,31,0.05052266518274943
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,31,0.04228266576925913
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,63,0.05016533533732096
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,63,0.04170133173465729
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,127,0.05450133482615153
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,127,0.04580800235271454
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,255,0.08473066488901775
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,255,0.055813332398732506
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,511,0.08764266967773438
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,511,0.15288533767064413
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,1023,0.2890133261680603
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,1023,0.1546026666959127
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,3,0.009434666484594345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,7,0.009770666559537252
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,3,0.0107893335322539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,1,0.010581333190202713
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,1,0.010714666297038397
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,7,0.010928000013033548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,15,0.009162666896979014
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,15,0.010640000303586325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,31,0.010874666273593903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,31,0.01081066702802976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,63,0.009178666397929192
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,63,0.010645333677530289
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,127,0.010586666564146677
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,127,0.00891733355820179
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,255,0.010832000523805618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,255,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,511,0.010640000303586325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,511,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,1023,0.01071999967098236
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,1023,0.010725333044926325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,2047,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,2047,0.011055999745925268
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,4095,0.01314666618903478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,4095,0.013151999562978745
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,8191,0.015024000157912573
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,8191,0.01532799998919169
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,16383,0.01736533393462499
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,16383,0.017125333348910015
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,32767,0.03268799930810928
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,32767,0.018112000077962875
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,65535,0.05189333359400431
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,65535,0.034074666599432625
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,131071,0.08469866712888081
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,131071,0.05392000079154968
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,1,0.009359999870260557
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,1,0.010581333190202713
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,3,0.010122666756312052
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,3,0.008879999940594038
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,7,0.009525333220760027
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,7,0.010778666784365972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,15,0.010154666379094124
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,15,0.008992000172535578
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,31,0.009349333122372627
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,31,0.010821333775917688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,63,0.009850666547815004
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,63,0.008879999940594038
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,127,0.010512000570694605
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,127,0.00919999989370505
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,255,0.008890666688481966
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,255,0.010656000425418219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,511,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,511,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,1023,0.010602666685978571
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,1023,0.011098666737476984
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,2047,0.012986666212479273
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,2047,0.012805332740147909
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,4095,0.01321600005030632
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,4095,0.012949333836634954
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,8191,0.0169813334941864
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,8191,0.014991999914248785
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,16383,0.031557333966096245
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,16383,0.01743999992807706
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,32767,0.049866666396458946
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,32767,0.03348266581694285
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,65535,0.0846666693687439
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,65535,0.05036800106366476
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,131071,0.15453333655993143
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,131071,0.08752533793449402
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,1,0.009039999917149544
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,3,0.010832000523805618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,1,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,3,0.0107893335322539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,7,0.010741333166758219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,7,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,15,0.010661333799362183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,15,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,31,0.009125333279371262
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,31,0.010725333044926325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,63,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,63,0.009535999968647957
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,127,0.011493333925803503
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,255,0.009162666896979014
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,127,0.011407999942700068
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,255,0.009594666461149851
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,511,0.010725333044926325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,511,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,1023,0.010911999891201654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,1023,0.011178666104873022
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,2047,0.012821332861979803
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,2047,0.011205332974592844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,4095,0.014837333311637243
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,8191,0.029637334247430164
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,4095,0.013130666067202887
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,8191,0.015301333119471868
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,16383,0.04794666667779287
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,16383,0.03129599988460541
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,32767,0.04877866804599762
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,32767,0.08146666487058003
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,65535,0.0842186709245046
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,65535,0.15016532937685648
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,1,0.01071999967098236
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,131071,0.2876960039138794
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,1,0.010629333555698395
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,131071,0.15585066874821982
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,3,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,3,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,7,0.01081066702802976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,7,0.010677333921194077
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,15,0.010928000013033548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,15,0.011034666250149408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,31,0.010666667173306147
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,31,0.01073066641887029
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,63,0.013050666699806849
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,63,0.010981333752473196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,127,0.01062400018175443
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,127,0.01073066641887029
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,255,0.010677333921194077
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,255,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,511,0.011226666470368704
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,511,0.011333333949247995
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,1023,0.01099733387430509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,1023,0.012885333349307379
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,2047,0.013301332791646322
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,2047,0.013264000415802002
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,4095,0.01516266663869222
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,4095,0.029322666426499683
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,8191,0.04667733112970988
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,8191,0.031328000128269196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,16383,0.04783466458320618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,16383,0.08096533517042796
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,32767,0.14939733346303305
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,32767,0.08309866487979889
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,65535,0.15069866180419922
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,65535,0.2863573431968689
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,131071,0.5634400049845377
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,131071,0.2881173292795817
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,1,0.019029332945744198
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,1,0.01692266638080279
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,3,0.019173332800467808
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,3,0.018160000443458557
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,7,0.019071999937295914
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,7,0.017055999487638474
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,15,0.018954666952292126
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,15,0.01783466711640358
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,31,0.01903466631968816
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,31,0.01735466718673706
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,63,0.01926933353145917
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,127,0.01903466631968816
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,127,0.016965333372354507
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,63,0.018325333793958027
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,255,0.03109866629044215
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,255,0.017093333105246227
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,511,0.048613334695498146
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,1,0.029296000798543293
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,511,0.03221333275238673
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,1,0.027295999228954315
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,3,0.029301332930723827
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,3,0.027530667682488758
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,7,0.029605334003766377
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,7,0.02737066646416982
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,15,0.029285334050655365
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,15,0.027552001178264618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,31,0.027258666853109997
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,31,0.029605334003766377
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,63,0.029418667157491047
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,63,0.02731200059254964
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,127,0.03555200000603994
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,127,0.027466667195161183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,255,0.05120000243186951
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,255,0.035349334279696144
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,511,0.08598933617273967
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,1,0.05397866666316986
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,511,0.05297600229581197
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,1,0.0459199994802475
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,3,0.0539626677831014
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,3,0.04565866788228353
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,7,0.054245332876841225
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,15,0.05397333204746246
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,15,0.045781334241231285
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,7,0.04585599899291992
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,31,0.0547626664241155
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,31,0.04573333263397217
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,63,0.05406933526198069
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,63,0.04566933214664459
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,127,0.059258664647738137
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,127,0.05142933130264282
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,255,0.08899199962615967
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,255,0.060549333691596985
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,511,0.15845866998036703
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,1,0.08899733424186707
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,511,0.09341866771380107
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,1,0.07937066753705342
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,3,0.09017067154248555
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,3,0.07856533428033192
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,7,0.0888159970442454
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,7,0.0790719985961914
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,15,0.07870933413505554
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,15,0.09045333663622539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,31,0.09011733531951904
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,31,0.08067200084527333
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,63,0.09122666716575623
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,63,0.0795360008875529
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,127,0.0976106623808543
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,127,0.08237333099047343
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,255,0.1583573321501414
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,255,0.09912000099817912
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,511,0.29466134309768677
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,511,0.16134933630625406
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,1,0.029279999434947968
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,1,0.027221334477265675
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,3,0.029359998802344005
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,3,0.027530667682488758
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,7,0.027263998985290527
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,15,0.02922666569550832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,7,0.029701332251230877
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,15,0.027637332677841187
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,31,0.02923733244339625
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,31,0.02734400083621343
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,63,0.02769600103298823
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,63,0.029520000020662945
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,127,0.03586133321126302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,255,0.03530666728814443
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,255,0.05219733218352
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,127,0.02716800073782603
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,1,0.053488001227378845
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,1,0.04390933116277059
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,3,0.05293866495291392
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,3,0.04372799893220266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,7,0.04374399781227112
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,15,0.05362666646639506
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,15,0.04363733530044556
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,7,0.053674668073654175
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,31,0.05190399785836538
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,31,0.044031997521718345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,63,0.05195199946562449
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,63,0.04393066465854645
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,127,0.05811200042565664
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,127,0.05002133548259735
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,255,0.08874133229255676
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,255,0.05972800155480703
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,1,0.09526933232943217
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,3,0.09729599952697754
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,1,0.08899733424186707
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,3,0.08876799543698628
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,7,0.09530666470527649
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,7,0.08699199557304382
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,15,0.09750933448473613
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,31,0.09763200084368388
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,31,0.08761599659919739
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,15,0.0874826709429423
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,63,0.09883200128873189
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,63,0.08741333087285359
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,127,0.1066986620426178
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,127,0.09118933478991191
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,255,0.16692266861597696
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,1,0.1697333256403605
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,1,0.1460586686929067
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,255,0.10749866565068562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,3,0.14640532930692038
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,3,0.16845866044362387
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,7,0.16739199558893839
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,7,0.14590932925542197
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,15,0.1462506651878357
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,15,0.16894400119781494
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,31,0.16916267077128092
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,31,0.145797332127889
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,63,0.16992000738779703
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,63,0.14666133125623068
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,127,0.1507306694984436
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,127,0.18346667289733887
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,1,0.009754666437705358
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,1,0.008954666554927826
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,3,0.009018666421373686
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,7,0.009002666920423508
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,3,0.010341333225369453
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,7,0.008986666798591614
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,15,0.009119999905427298
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,15,0.010970667004585266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,31,0.009253333633144697
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,31,0.011359999577204386
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,63,0.008992000172535578
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,63,0.010672000547250112
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,127,0.00927466650803884
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,127,0.011002667248249054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,255,0.009493333597977957
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,255,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,511,0.010832000523805618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,511,0.010778666784365972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,1023,0.010608000059922537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,1023,0.011247999966144562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,2047,0.013093333691358566
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,2047,0.012949333836634954
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,4095,0.013066666821638743
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,8191,0.015487999965747198
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,4095,0.013264000415802002
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,8191,0.014997333288192749
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,16383,0.03186133255561193
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,16383,0.017093333105246227
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,32767,0.04993600149949392
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,32767,0.033146666983763375
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,65535,0.0851039985815684
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,65535,0.049957334995269775
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,1,0.011029332876205444
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,1,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,3,0.010735999792814255
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,3,0.010832000523805618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,7,0.010794666906197866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,7,0.011007999380429586
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,255,0.3035946687062581
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,15,0.010784000158309937
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,15,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,31,0.010165333126982054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,31,0.010816000401973724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,63,0.011322667201360067
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,127,0.010757333288590113
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,127,0.00921066664159298
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,255,0.010015999898314476
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,255,0.00916800027092298
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,255,0.185263991355896
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,511,0.011045332998037338
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,511,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,1023,0.011242666592200598
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,2047,0.012730666746695837
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,2047,0.012794667234023413
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,4095,0.014202666779359182
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,63,0.010693332801262537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,4095,0.013717333475748697
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,8191,0.02937600016593933
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,8191,0.016997333616018295
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,16383,0.047877331574757896
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,16383,0.030799999833106995
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,32767,0.08308266599973042
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,32767,0.050053333242734276
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,65535,0.08497599760691325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,65535,0.15345066785812378
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,1023,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,1,0.010549332946538925
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,1,0.011130666981140772
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,3,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,3,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,7,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,7,0.011050666371981302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,15,0.01091733326514562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,15,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,31,0.010597333312034607
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,31,0.010677333921194077
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,63,0.01032533310353756
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,63,0.010672000547250112
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,127,0.010714666297038397
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,127,0.010666667173306147
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,255,0.0107893335322539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,255,0.010618666807810465
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,511,0.01099733387430509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,511,0.01129066695769628
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,1023,0.013221333424250284
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,1023,0.01121066634853681
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,2047,0.013327999661366144
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,2047,0.013189333180586496
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,4095,0.029472000896930695
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,4095,0.01611199975013733
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,8191,0.04710400104522705
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,8191,0.030778666337331135
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,16383,0.048394665122032166
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,16383,0.08186666667461395
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,32767,0.14984533190727234
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,32767,0.08430400490760803
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,65535,0.28810666004816693
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,1,0.010629333555698395
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,65535,0.1542080044746399
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,1,0.011183999478816986
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,3,0.01126933346192042
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,3,0.010805333654085795
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,7,0.010832000523805618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,15,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,15,0.010687999427318573
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,7,0.01138666644692421
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,31,0.010768000036478043
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,31,0.011018666128317514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,63,0.010794666906197866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,63,0.011114666859308878
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,127,0.010981333752473196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,127,0.011066666493813196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,255,0.011125333607196808
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,255,0.011061333119869232
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,511,0.011125333607196808
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,511,0.011258666714032492
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,1023,0.012917333592971167
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,1023,0.013253333667914072
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,2047,0.014831999937693277
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,2047,0.029365333418051403
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,4095,0.0462719996770223
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,4095,0.030000001192092896
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,8191,0.0480373352766037
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,8191,0.08077333370844524
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,16383,0.08225599924723308
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,16383,0.15025599797566733
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,32767,0.1504693329334259
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,32767,0.285866657892863
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,1,0.009930666536092758
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,1,0.010549332946538925
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,65535,0.28648533423741657
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,65535,0.5620266596476237
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,3,0.009919999788204828
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,3,0.010821333775917688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,7,0.010805333654085795
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,15,0.009824000298976898
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,7,0.00895999992887179
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,15,0.010703999549150467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,31,0.010293333480755487
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,31,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,63,0.00922133338948091
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,63,0.010597333312034607
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,127,0.010687999427318573
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,127,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,255,0.010277333358923594
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,255,0.010725333044926325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,511,0.010746666540702185
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,511,0.011007999380429586
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,1023,0.01091733326514562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,1023,0.011194666226704916
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,2047,0.012639999389648438
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,2047,0.012879999975363413
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,4095,0.015013333410024643
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,4095,0.013088000317414602
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,8191,0.030368000268936157
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,8191,0.016943999876578648
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,16383,0.04782933493455251
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,16383,0.03180266668399175
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,32767,0.08366933465003967
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,32767,0.04920533299446106
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,1,0.00956266683836778
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,65535,0.15495999654134116
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,65535,0.08520000179608662
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,1,0.011546666423479715
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,3,0.008885333314538002
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,3,0.009226666763424873
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,7,0.01073066641887029
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,7,0.010640000303586325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,15,0.008949333180983862
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,15,0.012069333344697952
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,31,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,31,0.010794666906197866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,63,0.009072000160813332
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,63,0.009674666449427605
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,127,0.010640000303586325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,127,0.010832000523805618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,255,0.009162666896979014
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,255,0.010746666540702185
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,511,0.011402666568756104
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,511,0.01101333275437355
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,1023,0.012890666723251343
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,1023,0.012671999633312225
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,2047,0.013354666531085968
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,2047,0.012975999464591345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,4095,0.02991466720898946
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,4095,0.01562133307258288
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,8191,0.047775998711586
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,8191,0.0313226655125618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,16383,0.08250133196512859
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,16383,0.048453330993652344
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,32767,0.1541973352432251
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,32767,0.08463467160860698
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,1,0.010672000547250112
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,1,0.011002667248249054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,65535,0.2956106662750244
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,3,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,3,0.011007999380429586
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,65535,0.15546133120854697
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,7,0.010751999914646149
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,7,0.01099733387430509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,15,0.011146667102972666
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,31,0.010762666662534079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,15,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,31,0.010677333921194077
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,63,0.0107893335322539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,63,0.011274666835864386
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,127,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,127,0.011066666493813196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,255,0.011114666859308878
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,255,0.011077333241701126
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,511,0.011178666104873022
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,511,0.011087999989589056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,1023,0.013301332791646322
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,1023,0.013141332815090815
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,2047,0.029253333806991577
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,2047,0.015205333630243937
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,4095,0.046581332882245384
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,4095,0.030394665896892548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,8191,0.0806933343410492
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,8191,0.04826666911443075
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,16383,0.15056533614794412
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,16383,0.08377066254615784
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,32767,0.28693334261576336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,32767,0.15474133690198263
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,1,0.008992000172535578
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,3,0.00895999992887179
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,1,0.008992000172535578
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,65535,0.29362134138743085
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,3,0.009242666885256767
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,7,0.009248000259200731
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,65535,0.5634666681289673
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,15,0.009178666397929192
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,7,0.010698666175206503
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,15,0.009002666920423508
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,31,0.009136000027259191
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,63,0.009290666629870733
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,63,0.00927466650803884
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,31,0.009472000102202097
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,127,0.008874666566650072
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,127,0.010618666807810465
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,255,0.008992000172535578
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,511,0.0102613332370917
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,255,0.010762666662534079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,511,0.011125333607196808
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,1023,0.009205333267649015
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,1023,0.008938666433095932
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,2047,0.011050666371981302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,4095,0.010805333654085795
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,2047,0.010847999403874079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,4095,0.01098666712641716
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,8191,0.014917333920796713
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,8191,0.015157333264748255
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,16383,0.019050666441520054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,16383,0.019029332945744198
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,32767,0.02722666660944621
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,32767,0.025402667621771496
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,65535,0.030720000465710957
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,65535,0.0295413335164388
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,131071,0.031541332602500916
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,131071,0.03128000100453695
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,1,0.009349333122372627
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,1,0.008890666688481966
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,3,0.009359999870260557
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,3,0.009103999783595404
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,7,0.009712000067035357
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,7,0.009066666786869368
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,15,0.009312000125646591
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,15,0.008943999807039896
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,31,0.009114666531483332
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,31,0.009568000212311745
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,63,0.009162666896979014
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,63,0.008943999807039896
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,127,0.009119999905427298
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,127,0.0106133334338665
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,255,0.009621333330869675
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,255,0.009279999881982803
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,511,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,511,0.011114666859308878
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,1023,0.01073066641887029
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,1023,0.010928000013033548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,2047,0.01062400018175443
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,2047,0.011226666470368704
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,4095,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,4095,0.011424000064531961
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,8191,0.01515199989080429
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,8191,0.01515199989080429
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,16383,0.019530666371186573
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,16383,0.01903466631968816
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,32767,0.02130666623512904
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,32767,0.021344001094500225
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,65535,0.023391999304294586
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,65535,0.023157333334287006
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,131071,0.04299733539422353
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,131071,0.024735999604066212
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,1,0.00892800030608972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,1,0.01080000028014183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,3,0.009109333157539368
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,3,0.009173333023985228
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,7,0.010640000303586325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,7,0.010053333515922228
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,15,0.008896000062425932
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,15,0.01098666712641716
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,31,0.008938666433095932
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,31,0.009045333291093508
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,63,0.010656000425418219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,63,0.010672000547250112
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,127,0.00914666677514712
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,127,0.010784000158309937
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,255,0.009178666397929192
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,255,0.008912000184257826
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,511,0.011002667248249054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,511,0.010965333630641302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,1023,0.010591999938090643
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,1023,0.010928000013033548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,2047,0.010634666929642359
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,2047,0.010944000134865442
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,4095,0.011258666714032492
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,4095,0.01102399950226148
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,8191,0.015103999525308609
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,8191,0.016149333367745083
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,16383,0.017290666699409485
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,16383,0.01749333366751671
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,32767,0.019578666736682255
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,65535,0.02145066608985265
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,32767,0.019839999576409657
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,65535,0.03591466695070267
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,131071,0.05397866666316986
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,1,0.009093333035707474
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,1,0.010687999427318573
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,131071,0.03864533454179764
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,3,0.009018666421373686
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,3,0.00884799969693025
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,7,0.009109333157539368
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,7,0.009999999776482582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,15,0.008826666822036108
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,15,0.00897066667675972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,31,0.009322666873534521
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,31,0.00933333362142245
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,63,0.008799999952316284
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,63,0.008863999818762144
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,127,0.010735999792814255
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,255,0.00878399983048439
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,127,0.010138666878143946
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,255,0.009072000160813332
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,511,0.010725333044926325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,511,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,1023,0.010816000401973724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,1023,0.009637333452701569
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,2047,0.010794666906197866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,2047,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,4095,0.011450666934251785
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,4095,0.011226666470368704
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,8191,0.015194666882356008
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,8191,0.014943999548753103
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,16383,0.01913600042462349
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,16383,0.019637333850065868
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,32767,0.021402666966120403
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,32767,0.021589333812395733
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,65535,0.023152001202106476
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,65535,0.023541333774725597
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,131071,0.040336000422636666
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,131071,0.026015999416510265
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,1,0.009130666653315226
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,1,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,3,0.00915733352303505
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,3,0.00933333362142245
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,7,0.009072000160813332
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,15,0.009530666594703993
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,15,0.009648000200589498
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,7,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,31,0.00915733352303505
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,31,0.009119999905427298
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,63,0.010784000158309937
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,63,0.010965333630641302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,127,0.00960533320903778
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,127,0.008853333070874214
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,255,0.010703999549150467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,255,0.00897066667675972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,511,0.00980266680320104
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,511,0.01101333275437355
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,1023,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,1023,0.010938666760921478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,2047,0.010954666882753372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,2047,0.010981333752473196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,4095,0.010938666760921478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,4095,0.011168000598748526
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,8191,0.015098666151364645
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,8191,0.015098666151364645
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,16383,0.020069333414236706
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,16383,0.017093333105246227
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,32767,0.019178666174411774
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,32767,0.01924266666173935
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,65535,0.021210665504137676
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,65535,0.03651199986537298
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,131071,0.0555626650651296
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,131071,0.03713600089152654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,1,0.009130666653315226
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,1,0.010992000500361124
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,3,0.009114666531483332
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,3,0.010709332923094431
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,7,0.009653333574533463
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,7,0.010842667271693548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,15,0.009317333499590555
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,15,0.010698666175206503
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,31,0.008901333436369896
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,31,0.010911999891201654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,63,0.009893333539366722
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,63,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,127,0.009258666386206945
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,127,0.010629333555698395
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,255,0.010640000303586325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,255,0.01109333336353302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,511,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,511,0.010741333166758219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,1023,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,1023,0.0107893335322539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,2047,0.011045332998037338
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,2047,0.01126933346192042
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,4095,0.01302933320403099
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,4095,0.013210666676362356
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,8191,0.015237333873907724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,16383,0.01701333373785019
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,16383,0.015200000256299973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,8191,0.015285332997639975
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,32767,0.03365866591533025
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,32767,0.01739199956258138
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,65535,0.049786667029062905
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,65535,0.036320000886917114
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,131071,0.08481599887212117
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,131071,0.05261866748332977
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,1,0.009248000259200731
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,3,0.01109333336353302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,3,0.010832000523805618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,1,0.010853332777818045
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,7,0.009194666519761086
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,7,0.010869332899649939
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,15,0.00973866693675518
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,31,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,15,0.01118933285276095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,31,0.010618666807810465
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,63,0.010634666929642359
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,63,0.01091733326514562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,127,0.010661333799362183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,255,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,127,0.011306667079528173
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,255,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,511,0.011130666981140772
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,511,0.011173332730929056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,1023,0.012757333616415659
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,1023,0.012960000584522883
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,2047,0.01471466695268949
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,2047,0.013007999708255133
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,4095,0.029818666477998097
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,8191,0.03107200066248576
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,8191,0.047925333182017006
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,4095,0.01563199982047081
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,16383,0.08423466483751933
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,16383,0.04877866804599762
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,32767,0.1606933375199636
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,1,0.01071999967098236
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,32767,0.08520000179608662
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,1,0.010885333021481832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,3,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,3,0.010837333897749582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,7,0.010821333775917688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,7,0.011098666737476984
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,15,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,15,0.010885333021481832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,31,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,31,0.011152000476916632
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,63,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,63,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,127,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,255,0.011120000233252844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,127,0.011045332998037338
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,255,0.010970667004585266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,511,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,511,0.011434666812419891
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,1023,0.013013333082199097
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,1023,0.0129120002190272
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,2047,0.015082667271296183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,4095,0.047413334250450134
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,2047,0.029663999875386555
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,4095,0.030016000072161358
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,8191,0.08236800134181976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,8191,0.048138668139775596
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,16383,0.15358400344848633
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,16383,0.08435733119646709
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,1,0.01091733326514562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,1,0.011600000162919363
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,32767,0.29498666524887085
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,3,0.01099733387430509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,3,0.01268799975514412
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,32767,0.1546026666959127
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,7,0.012874666601419449
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,7,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,15,0.01166933278242747
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,31,0.012586666891972223
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,31,0.011221333096424738
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,15,0.012693333129088083
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,63,0.011909333368142446
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,127,0.012858666479587555
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,63,0.011343999455372492
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,255,0.011920000116030375
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,127,0.011594666788975397
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,255,0.010938666760921478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,511,0.012768000364303589
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,511,0.014848000059525171
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,1023,0.029530666768550873
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,1023,0.015253332753976187
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,2047,0.04730666677157084
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,2047,0.03123733401298523
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,4095,0.08290133376916249
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,4095,0.0483893354733785
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,8191,0.15035733580589294
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,8191,0.08516266942024231
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,16383,0.29099200169245404
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,16383,0.15708800156911215
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,1,0.009679999823371569
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,3,0.009296000003814697
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,1,0.009189333145817121
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,32767,0.5677386522293091
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,32767,0.30160532395044964
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,3,0.009893333539366722
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,7,0.009093333035707474
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,15,0.010239999741315842
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,31,0.008832000195980072
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,7,0.00891733355820179
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,15,0.010597333312034607
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,31,0.010735999792814255
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,63,0.009136000027259191
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,63,0.00915733352303505
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,127,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,127,0.010778666784365972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,255,0.009130666653315226
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,255,0.009039999917149544
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,511,0.010682666053374609
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,511,0.01073066641887029
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,1023,0.010661333799362183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,1023,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,2047,0.010735999792814255
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,2047,0.010735999792814255
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,4095,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,4095,0.011296000331640244
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,8191,0.015189333508412043
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,16383,0.01714666684468587
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,8191,0.015471999843915304
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,16383,0.01700266698996226
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,32767,0.01939733326435089
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,32767,0.01889066646496455
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,65535,0.03583466758330663
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,65535,0.02110933264096578
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,131071,0.05394133428732554
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,131071,0.03761066744724909
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,1,0.009706666693091393
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,1,0.009194666519761086
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,3,0.00892800030608972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,3,0.010458666831254959
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,7,0.009002666920423508
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,7,0.00955200009047985
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,15,0.010735999792814255
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,15,0.008890666688481966
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,31,0.008863999818762144
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,31,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,63,0.010821333775917688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,63,0.009301333377758661
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,127,0.009066666786869368
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,127,0.010656000425418219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,255,0.010757333288590113
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,255,0.00919999989370505
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,511,0.010314666976531347
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,511,0.011152000476916632
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,1023,0.010650667051474253
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,1023,0.010842667271693548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,2047,0.010938666760921478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,2047,0.010725333044926325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,4095,0.013178666432698568
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,4095,0.013088000317414602
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,8191,0.015306666493415833
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,8191,0.014965333044528961
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,16383,0.017279999951521557
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,16383,0.017103999853134155
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,32767,0.033786666889985405
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,32767,0.01721599946419398
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,65535,0.03478399912516276
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,65535,0.05201066533724467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,131071,0.08548266688982646
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,1,0.0100853331387043
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,131071,0.05309333403905233
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,1,0.010794666906197866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,3,0.010005333150426546
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,3,0.0138026662170887
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,7,0.009349333122372627
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,7,0.010698666175206503
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,15,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,15,0.011125333607196808
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,31,0.010805333654085795
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,31,0.010698666175206503
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,63,0.009653333574533463
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,63,0.010543999572594961
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,127,0.010735999792814255
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,127,0.011018666128317514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,255,0.009797333429257074
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,255,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,511,0.010778666784365972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,511,0.010762666662534079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,1023,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,1023,0.010837333897749582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,2047,0.012746666868527731
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,4095,0.014725333700577417
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,4095,0.012997332960367203
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,2047,0.013258667041858038
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,8191,0.017210666090250015
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,8191,0.015546667079130808
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,16383,0.03138133386770884
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,16383,0.01740266631046931
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,32767,0.049695998430252075
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,32767,0.03348266581694285
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,65535,0.08307200173536937
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,65535,0.05106133222579956
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,131071,0.08710933725039165
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,131071,0.15219733119010925
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,1,0.011039999624093374
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,3,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,3,0.010981333752473196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,1,0.011018666128317514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,7,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,7,0.010677333921194077
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,15,0.01073066641887029
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,15,0.011242666592200598
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,31,0.010965333630641302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,31,0.011045332998037338
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,63,0.010703999549150467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,63,0.011114666859308878
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,127,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,127,0.011296000331640244
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,255,0.01071999967098236
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,255,0.01129066695769628
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,511,0.011007999380429586
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,511,0.01126933346192042
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,1023,0.013450667262077332
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,1023,0.012981332838535309
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,2047,0.02941333254178365
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,2047,0.015290666371583939
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,4095,0.04773333172003428
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,8191,0.08339732885360718
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,4095,0.03030933439731598
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,8191,0.04866133133570353
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,16383,0.08437333504358928
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,1,0.011194666226704916
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,1,0.011178666104873022
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,16383,0.1546933352947235
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,3,0.010944000134865442
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,3,0.011370666325092316
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,7,0.012437333663304647
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,7,0.01250133290886879
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,15,0.010928000013033548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,15,0.010992000500361124
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,31,0.011887999872366587
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,31,0.011264000087976456
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,63,0.01097600037852923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,63,0.01119999960064888
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,127,0.012538666526476542
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,127,0.011258666714032492
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,255,0.012560000022252401
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,255,0.010965333630641302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,511,0.014853333433469137
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,511,0.013056000073750814
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,1023,0.014949332922697067
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,1023,0.029946667452653248
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,2047,0.04797333478927612
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,2047,0.0313226655125618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,4095,0.04986133178075155
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,4095,0.08374933401743571
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,8191,0.1565600037574768
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,8191,0.08641599615414937
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,16383,0.2962239980697632
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,16383,0.15788267056147257
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,1,0.014917333920796713
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,1,0.01333333303531011
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,3,0.015226667126019796
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,3,0.013327999661366144
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,7,0.014917333920796713
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,7,0.014874666929244995
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,15,0.013461332768201828
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,15,0.015392000476519266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,31,0.014991999914248785
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,31,0.015066667149464289
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,63,0.01515199989080429
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,127,0.015103999525308609
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,127,0.014933332800865173
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,63,0.013306666165590286
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,255,0.015008000036080679
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,255,0.013023999830087027
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,511,0.029605334003766377
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,511,0.01706133286158244
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,1023,0.031317333380381264
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,1023,0.04836800197760264
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,2047,0.08374933401743571
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,2047,0.04994666576385498
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,4095,0.1540426711241404
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,4095,0.08636800448099773
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,8191,0.2951093316078186
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,8191,0.1574666698773702
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,16383,0.5773066679636637
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,1,0.012645332763592402
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,16383,0.2977760036786397
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,1,0.011077333241701126
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,3,0.012885333349307379
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,3,0.01146666705608368
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,7,0.011050666371981302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,7,0.01293333371480306
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,15,0.012917333592971167
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,15,0.011029332876205444
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,31,0.012863999853531519
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,31,0.012906666845083237
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,63,0.01302933320403099
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,63,0.012602667013804117
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,127,0.012890666723251343
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,127,0.012874666601419449
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,255,0.01293333371480306
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,255,0.012821332861979803
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,511,0.015018666783968607
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,511,0.0136266661187013
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,1023,0.029733332494894665
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,1023,0.016901332885026932
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,2047,0.048623998959859215
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,2047,0.03164800008138021
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,4095,0.08444266517957051
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,4095,0.05008000135421753
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,1,0.014746667196353277
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,1,0.013253333667914072
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,7,0.014991999914248785
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,3,0.015024000157912573
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,3,0.013157332936922709
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,7,0.013610667238632837
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,15,0.014954666296641031
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,15,0.013210666676362356
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,31,0.014725333700577417
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,31,0.013157332936922709
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,63,0.015210667004187902
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,63,0.013002666334311167
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,127,0.014965333044528961
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,127,0.013418667018413544
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,255,0.01481066644191742
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,255,0.013221333424250284
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,511,0.030005333324273426
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,511,0.01746133342385292
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,1023,0.04825599988301595
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,1023,0.031541332602500916
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,2047,0.08459200461705525
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,2047,0.04971200227737427
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,4095,0.15567466616630554
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,4095,0.08598933617273967
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,1,0.01921066641807556
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,3,0.01904533306757609
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,3,0.017071999609470367
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,1,0.017562666287024815
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,7,0.018863999595244724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,7,0.017114666601022083
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,15,0.019029332945744198
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,15,0.017610666652520496
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,31,0.01884799947341283
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,31,0.01708799973130226
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,63,0.019141333798567455
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,63,0.017504000415404636
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,127,0.019098666807015736
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,127,0.01704000060757001
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,255,0.02941333254178365
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,255,0.017360000560681026
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,511,0.04770666857560476
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,511,0.031397332747777305
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,1023,0.08148266871770223
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,1023,0.049440001447995506
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,2047,0.0841919978459676
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,2047,0.15042666594187418
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,4095,0.28676267464955646
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,4095,0.1552906632423401
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,1,0.01509333277742068
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,1,0.013072000195582708
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,3,0.014725333700577417
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,3,0.014805333067973455
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,7,0.013007999708255133
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,7,0.015168000012636185
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,15,0.015024000157912573
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,15,0.0163680004576842
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,31,0.015077333897352219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,63,0.015072000523408255
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,31,0.013327999661366144
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,63,0.01492799942692121
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,127,0.015008000036080679
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,127,0.012965332716703415
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,255,0.015285332997639975
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,255,0.014869333555301031
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,511,0.030618667602539062
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,511,0.017136000096797943
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,1023,0.031541332602500916
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,1023,0.04987200101216634
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,2047,0.08587732911109924
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,2047,0.050213331977526345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,1,0.018981333822011948
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,1,0.017477333545684814
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,3,0.019066666563351948
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,3,0.017077332983414333
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,7,0.019013332823912304
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,7,0.017605333278576534
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,15,0.018976000448067982
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,15,0.017349333812793095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,31,0.01923199991385142
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,31,0.01754133279124896
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,63,0.019088000059127808
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,63,0.017162666966517765
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,127,0.018805333723624546
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,127,0.017322666943073273
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,255,0.02995733420054118
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,255,0.017386666188637417
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,511,0.04794133206208547
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,511,0.03152533372243246
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,1023,0.08268266419569652
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,1023,0.049456000328063965
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,2047,0.15391467014948526
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,1,0.029391999046007793
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,2047,0.08547733227411906
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,1,0.027082666754722595
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,3,0.0273333340883255
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,3,0.029626667499542236
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,7,0.027242665489514668
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,7,0.02951466788848241
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,15,0.02940266579389572
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,15,0.027535999814669292
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,31,0.029450667401154835
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,31,0.02754666656255722
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,63,0.029333333174387615
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,63,0.027530667682488758
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,127,0.034901333351929985
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,127,0.027647999425729115
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,255,0.0499946673711141
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,255,0.03588266670703888
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,511,0.0846720039844513
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,511,0.05221333106358846
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,1023,0.15275200208028158
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,1023,0.0870293378829956
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,1,0.008890666688481966
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,3,0.008863999818762144
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,2047,0.28886399666468304
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,3,0.009114666531483332
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,7,0.008922666932145754
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,7,0.00903466654320558
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,1,0.009375999992092451
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,15,0.00943999985853831
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,15,0.009061333412925402
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,31,0.009919999788204828
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,31,0.008896000062425932
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,63,0.008954666554927826
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,63,0.010762666662534079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,2047,0.15621333320935568
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,127,0.008832000195980072
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,127,0.00916800027092298
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,255,0.009712000067035357
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,255,0.009194666519761086
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,511,0.010928000013033548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,1023,0.01097600037852923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,511,0.010741333166758219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,2047,0.010981333752473196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,1023,0.010970667004585266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,2047,0.011077333241701126
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,8191,0.015237333873907724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,4095,0.013445333888133367
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,4095,0.013487999637921652
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,8191,0.015295999745527903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,16383,0.0173333336909612
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,32767,0.033471999069054924
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,16383,0.017045332739750545
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,32767,0.018805333723624546
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,65535,0.052015999952952065
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,65535,0.03514666606982549
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,131071,0.08659199873606364
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,1,0.009882666791478792
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,131071,0.05215999980767568
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,1,0.01080000028014183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,3,0.010490667074918747
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,3,0.009162666896979014
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,7,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,7,0.010832000523805618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,15,0.008965333302815756
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,31,0.009989333028594652
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,15,0.010602666685978571
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,31,0.009173333023985228
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,63,0.010255999863147736
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,63,0.010677333921194077
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,127,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,127,0.009082666908701261
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,255,0.010709332923094431
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,255,0.01091733326514562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,511,0.010837333897749582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,511,0.011109333485364914
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,1023,0.010832000523805618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,1023,0.010992000500361124
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,2047,0.01309866706530253
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,2047,0.012938667088747025
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,4095,0.013301332791646322
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,4095,0.013061333447694778
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,8191,0.015040000279744467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,8191,0.016858667135238647
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,16383,0.03162666658560435
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,16383,0.017157333592573803
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,32767,0.05002133548259735
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,32767,0.032986665765444435
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,65535,0.04985600213209788
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,65535,0.08569066723187764
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,1,0.009088000282645226
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,131071,0.15445866187413534
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,131071,0.08711466193199158
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,1,0.0122079998254776
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,3,0.010559999694426855
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,3,0.010709332923094431
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,7,0.008997333546479544
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,7,0.010944000134865442
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,15,0.010842667271693548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,31,0.008922666932145754
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,15,0.011002667248249054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,31,0.011215999722480774
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,63,0.010885333021481832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,63,0.009183999771873156
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,127,0.01071999967098236
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,255,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,127,0.011077333241701126
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,255,0.009461333354314169
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,511,0.011194666226704916
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,511,0.010970667004585266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,1023,0.010965333630641302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,1023,0.011173332730929056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,2047,0.0129120002190272
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,2047,0.011002667248249054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,4095,0.01443733274936676
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,4095,0.013397333522637686
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,8191,0.015200000256299973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,8191,0.029861333469549816
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,16383,0.04717333118120829
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,16383,0.03365866591533025
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,32767,0.08081066608428955
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,32767,0.048165331284205117
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,65535,0.15067199865976968
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,65535,0.0841919978459676
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,131071,0.2874186635017395
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,131071,0.15734400351842245
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,1,0.018992000569899876
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,1,0.017535999417304993
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,3,0.018981333822011948
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,3,0.01730666682124138
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,7,0.019130667050679524
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,15,0.019120000302791595
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,7,0.01741333305835724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,15,0.017738666385412216
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,31,0.01950399950146675
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,31,0.017077332983414333
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,63,0.018901333212852478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,63,0.01749333366751671
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,127,0.01923199991385142
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,127,0.017082666357358296
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,255,0.029845332105954487
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,255,0.01728533332546552
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,511,0.04833599925041199
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,511,0.03166933357715607
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,1023,0.08489599823951721
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,1023,0.049813335140546165
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,1,0.02921066681543986
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,1,0.027295999228954315
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,3,0.02961066613594691
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,3,0.02737066646416982
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,7,0.02932800104220708
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,7,0.027786667148272198
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,15,0.029653333127498627
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,31,0.029322666426499683
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,15,0.02718399961789449
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,31,0.02735466758410136
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,63,0.02962133288383484
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,63,0.02739199995994568
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,127,0.03472000112136205
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,255,0.050341332952181496
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,127,0.0271573339899381
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,255,0.03557866563399633
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,511,0.08571199576059978
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,511,0.05287999908129374
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,1023,0.15524799625078836
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,1,0.05389333268006643
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,1023,0.08852799733479817
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,1,0.04602666695912679
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,3,0.053786665201187134
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,3,0.043978666265805565
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,7,0.0543093333641688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,7,0.045519997676213585
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,15,0.05389333268006643
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,15,0.04371733466784159
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,31,0.04571733375390371
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,31,0.05411200225353241
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,63,0.054666668176651
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,63,0.044010668992996216
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,127,0.059104000528653465
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,127,0.052015999952952065
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,255,0.088837335507075
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,255,0.060032000144322716
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,511,0.1569706698258718
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,511,0.09636267026265462
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,1023,0.2937653263409932
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,1023,0.16236266493797302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,1,0.02941333254178365
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,1,0.02720533311367035
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,3,0.02940800040960312
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,3,0.02735466758410136
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,7,0.029637334247430164
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,7,0.027317332724730175
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,15,0.029370665550231934
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,15,0.027285332481066387
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,31,0.02956266701221466
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,31,0.027285332481066387
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,63,0.02941333254178365
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,63,0.027141332626342773
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,127,0.035674666364987694
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,255,0.050154666105906166
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,127,0.02770666778087616
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,255,0.03541333228349686
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,511,0.08587732911109924
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,1,0.05268799761931101
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,511,0.05225066840648651
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,1,0.043696001172065735
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,7,0.05198933184146881
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,7,0.043696001172065735
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,3,0.05268266797065735
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,3,0.044351999958356224
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,15,0.05221866567929586
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,15,0.04397333165009817
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,31,0.0528106689453125
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,31,0.04401599864164988
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,63,0.04379733403523763
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,127,0.05825600028038025
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,63,0.05266666909058889
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,127,0.04969066878159841
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,255,0.08866666754086812
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,255,0.05787733197212219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,511,0.15844800074895224
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,511,0.09147199988365173
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,1,0.09702400366465251
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,1,0.08796800176302592
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,3,0.09769599636395772
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,7,0.09738133351008098
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,3,0.08889066179593404
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,7,0.08806399504343669
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,15,0.09730666875839233
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,15,0.08733866612116496
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,31,0.09750399986902873
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,31,0.08767466743787129
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,63,0.09762666622797649
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,63,0.08714666962623596
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,127,0.10574400424957275
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,127,0.09098133444786072
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,255,0.16451199849446616
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,255,0.1081813375155131
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,1,0.00949866697192192
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,1,0.010245333115259806
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,3,0.010138666878143946
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,7,0.009434666484594345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,7,0.008896000062425932
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,3,0.009018666421373686
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,15,0.009098666409651438
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,15,0.008922666932145754
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,31,0.009189333145817121
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,31,0.009189333145817121
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,63,0.009141333401203156
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,63,0.00921066664159298
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,127,0.008869333192706108
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,127,0.009162666896979014
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,255,0.009002666920423508
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,255,0.008933333059151968
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,511,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,511,0.010741333166758219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,1023,0.010965333630641302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,1023,0.011029332876205444
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,2047,0.012965332716703415
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,2047,0.012965332716703415
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,4095,0.013546666751305262
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,4095,0.014597332725922266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,8191,0.015311999867359797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,8191,0.015013333410024643
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,16383,0.0317546675602595
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,32767,0.05003733436266581
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,16383,0.017658667018016178
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,32767,0.03305066625277201
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,65535,0.08504000306129456
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,65535,0.05108266572157542
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,131071,0.08638399839401245
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,131071,0.15633599956830344
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,1,0.010853332777818045
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,1,0.011114666859308878
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,3,0.010869332899649939
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,3,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,7,0.010415999839703241
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,7,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,15,0.008901333436369896
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,31,0.008997333546479544
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,15,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,31,0.00915733352303505
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,63,0.01073066641887029
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,63,0.009568000212311745
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,511,0.3015999992688497
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,127,0.01080000028014183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,127,0.009338666374484697
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,255,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,255,0.009253333633144697
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,511,0.010768000036478043
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,1023,0.010847999403874079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,1023,0.01102399950226148
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,2047,0.012714666624863943
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,511,0.17239999771118164
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,2047,0.011215999722480774
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,4095,0.01505600040157636
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,4095,0.013274667163689932
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,8191,0.015040000279744467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,16383,0.04799466828505198
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,16383,0.03109866629044215
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,32767,0.08293333152929942
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,32767,0.0491893341143926
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,511,0.010762666662534079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,65535,0.15436266859372458
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,65535,0.0846026639143626
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,1,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,131071,0.29526933034261066
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,1,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,131071,0.15530666708946228
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,8191,0.029557332396507263
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,3,0.010543999572594961
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,3,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,7,0.010053333515922228
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,15,0.01071999967098236
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,7,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,15,0.010138666878143946
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,31,0.010965333630641302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,31,0.010970667004585266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,63,0.010581333190202713
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,63,0.010522666076819101
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,127,0.010682666053374609
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,255,0.010874666273593903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,127,0.010698666175206503
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,255,0.01055466632048289
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,511,0.01081066702802976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,511,0.01116266722480456
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,1023,0.011173332730929056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,1023,0.012810666114091873
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,2047,0.013194666554530462
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,2047,0.013199999928474426
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,4095,0.029626667499542236
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,4095,0.017082666357358296
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,8191,0.047050664822260536
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,8191,0.03166933357715607
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,16383,0.08146133522192638
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,16383,0.049839998284975685
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,32767,0.08432533343633015
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,32767,0.15041066209475198
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,65535,0.2876586715380351
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,65535,0.1564640005429586
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,131071,0.2942986687024434
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,1,0.00927466650803884
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,1,0.010794666906197866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,131071,0.5631093184153239
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,3,0.009098666409651438
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,3,0.008933333059151968
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,7,0.009824000298976898
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,7,0.010693332801262537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,15,0.008879999940594038
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,15,0.010837333897749582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,31,0.010474666953086853
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,31,0.010832000523805618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,63,0.00892800030608972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,63,0.010762666662534079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,127,0.010805333654085795
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,127,0.010693332801262537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,255,0.008912000184257826
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,255,0.010751999914646149
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,511,0.010832000523805618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,511,0.010992000500361124
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,1023,0.011343999455372492
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,1023,0.01129066695769628
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,2047,0.012917333592971167
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,2047,0.012725333372751871
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,4095,0.014949332922697067
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,4095,0.01303999995191892
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,8191,0.03012799968322118
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,8191,0.016074666132529575
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,16383,0.04790399968624115
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,16383,0.031285333136717476
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,32767,0.08418666323026021
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,32767,0.04881600042184194
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,65535,0.1548533340295156
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,65535,0.08548266688982646
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,1,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,1,0.011050666371981302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,131071,0.2975733280181885
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,3,0.010698666175206503
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,3,0.010847999403874079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,7,0.01098666712641716
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,131071,0.15624533096949259
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,7,0.010757333288590113
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,15,0.010885333021481832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,15,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,31,0.01097600037852923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,31,0.010714666297038397
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,63,0.010992000500361124
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,63,0.01101333275437355
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,127,0.01091733326514562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,127,0.010816000401973724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,255,0.010938666760921478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,255,0.010928000013033548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,511,0.010885333021481832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,511,0.011338666081428528
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,1023,0.013594667116800943
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,1023,0.012362666428089142
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,2047,0.013376000026861826
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,2047,0.013258667041858038
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,4095,0.029317334294319153
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,4095,0.01523200049996376
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,8191,0.04786133269468943
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,8191,0.031285333136717476
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,16383,0.08307733138402303
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,16383,0.04821333289146423
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,32767,0.1536960005760193
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,32767,0.08478933572769165
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,65535,0.15574933091799417
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,65535,0.2955626646677653
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,1,0.008890666688481966
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,1,0.010874666273593903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,3,0.009136000027259191
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,3,0.010794666906197866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,131071,0.5768533150355021
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,7,0.008933333059151968
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,7,0.010672000547250112
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,15,0.009061333412925402
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,131071,0.2969813346862793
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,31,0.010853332777818045
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,15,0.008901333436369896
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,31,0.009109333157539368
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,63,0.01089599976936976
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,127,0.008853333070874214
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,127,0.009050666665037474
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,63,0.010687999427318573
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,255,0.01080000028014183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,255,0.008853333070874214
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,511,0.010821333775917688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,511,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,1023,0.00903466654320558
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,1023,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,2047,0.01102399950226148
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,2047,0.010672000547250112
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,4095,0.010821333775917688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,4095,0.011690666278203329
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,8191,0.015130666395028433
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,16383,0.019023999571800232
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,8191,0.015072000523408255
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,32767,0.025258667767047882
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,16383,0.01974933346112569
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,32767,0.025888000925381977
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,65535,0.03133333226044973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,65535,0.029631999631722767
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,131071,0.033786666889985405
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,131071,0.031712000568707786
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,1,0.008922666932145754
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,1,0.010842667271693548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,3,0.011087999989589056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,3,0.009248000259200731
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,7,0.00897066667675972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,7,0.010794666906197866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,15,0.009743999689817429
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,15,0.009242666885256767
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,31,0.00901333304742972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,31,0.010970667004585266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,63,0.009850666547815004
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,63,0.00895999992887179
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,127,0.008912000184257826
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,255,0.009162666896979014
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,127,0.010885333021481832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,255,0.009343999748428663
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,511,0.010853332777818045
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,511,0.01146666705608368
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,1023,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,1023,0.011077333241701126
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,2047,0.011354666203260422
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,2047,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,4095,0.01102399950226148
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,8191,0.015173333386580149
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,4095,0.01163200040658315
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,8191,0.015013333410024643
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,16383,0.019445333629846573
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,16383,0.01960533360640208
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,32767,0.021669333179791767
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,32767,0.021269333859284718
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,65535,0.02378133436044057
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,65535,0.02346666653951009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,131071,0.04301333427429199
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,131071,0.025807999074459076
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,1,0.009008000294367472
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,1,0.009312000125646591
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,3,0.01139733319481214
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,3,0.009338666374484697
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,7,0.009152000149091085
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,15,0.009269333134094873
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,7,0.011296000331640244
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,15,0.009242666885256767
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,31,0.009359999870260557
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,31,0.011301333705584208
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,63,0.009226666763424873
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,63,0.00891733355820179
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,127,0.009141333401203156
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,127,0.009877333417534828
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,255,0.009077333534757296
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,255,0.009072000160813332
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,511,0.011178666104873022
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,511,0.010992000500361124
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,1023,0.009402666861812273
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,1023,0.011237333218256632
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,2047,0.010602666685978571
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,2047,0.01110400011142095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,4095,0.011120000233252844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,4095,0.011685332904259363
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,8191,0.01498666654030482
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,16383,0.019093333433071773
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,8191,0.015216000378131866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,16383,0.019925333559513092
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,32767,0.02149333308140437
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,32767,0.021317332983016968
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,65535,0.02499199906984965
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,131071,0.040405333042144775
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,65535,0.02364266663789749
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,131071,0.02534399926662445
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,1,0.0106133334338665
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,1,0.00901333304742972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,3,0.011242666592200598
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,3,0.008901333436369896
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,7,0.009941333283980688
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,7,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,15,0.011194666226704916
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,15,0.009343999748428663
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,31,0.00914666677514712
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,31,0.010330666477481524
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,63,0.0099093330403169
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,63,0.011136000355084738
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,127,0.009136000027259191
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,127,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,255,0.00919999989370505
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,255,0.011194666226704916
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,511,0.010981333752473196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,511,0.010746666540702185
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,1023,0.010874666273593903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,1023,0.011077333241701126
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,2047,0.010794666906197866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,4095,0.010981333752473196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,2047,0.01110400011142095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,4095,0.011589333415031433
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,8191,0.015050667027632395
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,8191,0.014997333288192749
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,16383,0.017658667018016178
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,16383,0.017498667041460674
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,32767,0.019296000401178997
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,32767,0.019205333044131596
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,65535,0.03617066641648611
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,65535,0.02163200080394745
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,131071,0.03730133424202601
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,131071,0.054373333851496376
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,1,0.010693332801262537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,1,0.011146667102972666
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,3,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,3,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,7,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,15,0.009269333134094873
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,7,0.01102399950226148
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,15,0.010928000013033548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,31,0.01062400018175443
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,31,0.010725333044926325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,63,0.011066666493813196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,63,0.010869332899649939
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,127,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,127,0.011413333316644033
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,255,0.010010666524370512
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,255,0.010687999427318573
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,511,0.011621333658695221
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,511,0.010965333630641302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,1023,0.012768000364303589
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,1023,0.012608000387748083
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,2047,0.013653332988421122
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,2047,0.013258667041858038
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,4095,0.03065599997838338
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,8191,0.04796266555786133
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,8191,0.03152533372243246
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,4095,0.016234666109085083
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,16383,0.04934399823347727
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,16383,0.08515200018882751
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,32767,0.15551466743151346
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,32767,0.08527466654777527
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,1,0.0107893335322539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,65535,0.29681599140167236
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,1,0.01098666712641716
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,65535,0.15718932946523032
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,3,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,3,0.011061333119869232
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,7,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,7,0.011173332730929056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,15,0.010703999549150467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,15,0.010784000158309937
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,31,0.010981333752473196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,31,0.011173332730929056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,63,0.01080000028014183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,63,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,127,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,127,0.011887999872366587
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,255,0.010768000036478043
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,255,0.010970667004585266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,511,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,511,0.011557333171367645
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,1023,0.013317332913478216
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,1023,0.012752000242471695
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,2047,0.029461334149042766
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,2047,0.01581866666674614
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,4095,0.047775998711586
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,4095,0.029445332785447437
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,8191,0.08180266618728638
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,8191,0.04831466575463613
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,16383,0.08455466230710347
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,16383,0.1530453364054362
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,32767,0.29443732897440594
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,1,0.009029333169261614
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,32767,0.15558933218320212
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,1,0.009253333633144697
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,65535,0.5771786769231161
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,65535,0.29577600955963135
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,3,0.009253333633144697
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,3,0.009045333291093508
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,15,0.008890666688481966
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,15,0.009813333551088968
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,31,0.009109333157539368
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,31,0.011087999989589056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,7,0.009232000137368837
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,7,0.008943999807039896
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,63,0.009237333511312803
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,63,0.011296000331640244
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,127,0.009008000294367472
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,127,0.009130666653315226
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,255,0.010741333166758219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,255,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,511,0.01091733326514562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,511,0.010698666175206503
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,1023,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,2047,0.010954666882753372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,1023,0.011231999844312668
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,2047,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,4095,0.011055999745925268
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,4095,0.010869332899649939
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,8191,0.015647999942302704
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,16383,0.01703466723362605
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,8191,0.015461333096027374
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,16383,0.01706133286158244
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,32767,0.01969066634774208
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,32767,0.01912533367673556
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,65535,0.03649600098530451
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,65535,0.020234666764736176
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,131071,0.037808001041412354
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,131071,0.05589866638183594
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,1,0.009392000113924345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,1,0.009343999748428663
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,3,0.009141333401203156
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,3,0.010079999764760336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,7,0.00915733352303505
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,7,0.011002667248249054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,15,0.009077333534757296
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,15,0.011034666250149408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,31,0.00916800027092298
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,31,0.01071999967098236
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,63,0.009162666896979014
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,63,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,127,0.01219733307758967
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,127,0.009002666920423508
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,255,0.009712000067035357
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,255,0.009477333476146063
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,511,0.011146667102972666
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,511,0.011328000575304031
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,1023,0.010565333068370819
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,1023,0.010970667004585266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,2047,0.011365332951148352
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,2047,0.011120000233252844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,4095,0.013088000317414602
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,4095,0.013797332843144735
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,8191,0.01526933287580808
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,8191,0.015530666957298914
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,16383,0.01533866673707962
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,16383,0.018197332819302876
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,32767,0.03373866776625315
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,32767,0.01735466718673706
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,65535,0.03374933451414108
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,65535,0.05266133447488149
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,131071,0.08539733290672302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,131071,0.05299200117588043
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,1,0.011519999553759893
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,1,0.012389333297808966
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,3,0.010965333630641302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,3,0.01128000020980835
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,7,0.010879999647537867
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,15,0.011050666371981302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,7,0.012096000214417776
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,15,0.01119999960064888
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,31,0.011098666737476984
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,31,0.01097600037852923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,127,0.010885333021481832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,63,0.011589333415031433
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,127,0.011039999624093374
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,63,0.011887999872366587
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,255,0.011125333607196808
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,255,0.011440000186363855
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,511,0.01101333275437355
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,511,0.011445333560307821
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,1023,0.0129120002190272
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,2047,0.029205332199732464
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,1023,0.01321600005030632
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,8191,0.08418132861455281
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,2047,0.015290666371583939
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,4095,0.030645333230495453
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,4095,0.047637333472569786
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,8191,0.049733335773150124
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,16383,0.15550933281580606
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,16383,0.08519466718037923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,32767,0.15638933579126993
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,32767,0.29891733328501385
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,1,0.011205332974592844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,1,0.012831999609867731
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,3,0.012538666526476542
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,3,0.011237333218256632
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,7,0.013274667163689932
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,7,0.012698666503032049
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,15,0.012560000022252401
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,15,0.011322667201360067
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,31,0.013183999806642532
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,31,0.011578666667143503
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,63,0.012794667234023413
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,63,0.011231999844312668
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,127,0.01251199965675672
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,127,0.012901333471139273
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,255,0.011237333218256632
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,255,0.01312000056107839
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,511,0.015061333775520325
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,511,0.013173333058754602
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,1023,0.029535998900731403
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,1023,0.017157333592573803
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,2047,0.030853333572546642
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,2047,0.0480320006608963
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,4095,0.08381332953770955
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,4095,0.050288001696268715
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,16383,0.2979573408762614
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,8191,0.08691199620564778
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,8191,0.15380266308784485
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,16383,0.1570026675860087
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,32767,0.5789813200632731
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,1,0.01128000020980835
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,32767,0.30301332473754883
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,1,0.011514666179815928
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,3,0.012917333592971167
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,3,0.011488000551859537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,7,0.013007999708255133
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,7,0.011349332829316458
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,15,0.011141333729028702
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,15,0.011402666568756104
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,31,0.013034666577974955
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,63,0.01129066695769628
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,31,0.011760000139474869
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,63,0.011183999478816986
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,127,0.013253333667914072
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,127,0.011498666057984034
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,255,0.013434667140245438
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,255,0.011173332730929056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,511,0.01523200049996376
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,511,0.013002666334311167
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,1023,0.030975999931494396
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,1023,0.017301333447297413
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,2047,0.048351998130480446
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,2047,0.032144000132878624
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,8191,0.15537066260973612
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,4095,0.05020800232887268
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,4095,0.08544533451398213
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,8191,0.08610666791598003
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,1,0.015034666905800501
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,1,0.013562666873137156
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,3,0.015210667004187902
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,7,0.014309333016475042
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,3,0.015392000476519266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,7,0.013557333499193192
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,15,0.015189333508412043
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,15,0.013248000293970108
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,31,0.015279999623696009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,31,0.013370666652917862
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,63,0.015439999600251516
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,63,0.013280000537633896
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,127,0.015392000476519266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,127,0.013754667093356451
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,255,0.01339200014869372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,255,0.015461333096027374
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,511,0.03070399910211563
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,511,0.017301333447297413
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,1023,0.04851733148097992
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,1023,0.031541332602500916
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,4095,0.15638933579126993
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,2047,0.08515200018882751
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,4095,0.08581866820653279
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,2047,0.050250664353370667
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,8191,0.29978134234746295
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,8191,0.15823466579119363
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,1,0.015103999525308609
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,1,0.015029333531856537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,3,0.01505600040157636
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,7,0.014991999914248785
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,3,0.013477332890033722
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,7,0.015050667027632395
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,15,0.014741333822409311
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,15,0.013130666067202887
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,31,0.01524266724785169
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,31,0.015642666568358738
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,63,0.015184000134468079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,63,0.013408000270525614
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,127,0.015370666980743408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,127,0.013898666948080063
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,255,0.015087999403476715
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,255,0.01341333364446958
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,511,0.03162133445342382
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,511,0.017770666629076004
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,1023,0.04865066707134247
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,1023,0.03178666780392329
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,2047,0.08545600374539693
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,2047,0.050800000627835594
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,4095,0.15824000040690103
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,4095,0.08640000224113464
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,1,0.01932799940307935
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,1,0.017685333887736004
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,3,0.0220320001244545
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,7,0.01933866615096728
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,3,0.017829333742459614
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,7,0.017840000490347546
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,15,0.01916266605257988
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,15,0.01754666616519292
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,31,0.019813333948453266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,63,0.019215999792019527
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,31,0.017759999881188076
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,63,0.017525333911180496
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,127,0.019600000232458115
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,127,0.01720000058412552
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,255,0.03192000091075897
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,255,0.017621333400408428
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,511,0.048101335763931274
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,511,0.031066666046778362
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,1023,0.08329066634178162
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,1023,0.049173335234324135
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,2047,0.15343999862670898
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,2047,0.08458133538564046
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,1,0.009338666374484697
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,4095,0.2922719915707906
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,1,0.009077333534757296
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,3,0.00922133338948091
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,7,0.00903466654320558
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,7,0.009178666397929192
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,3,0.011168000598748526
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,15,0.009098666409651438
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,15,0.009402666861812273
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,31,0.011306667079528173
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,31,0.009066666786869368
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,63,0.009317333499590555
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,63,0.011173332730929056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,4095,0.15595733126004538
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,127,0.009312000125646591
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,127,0.010933333386977514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,255,0.011253333340088526
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,255,0.009093333035707474
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,511,0.011130666981140772
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,511,0.011002667248249054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,1023,0.011402666568756104
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,1023,0.011253333340088526
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,2047,0.011007999380429586
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,2047,0.011231999844312668
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,4095,0.013493333011865616
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,4095,0.014314666390419006
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,8191,0.01586666703224182
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,8191,0.015466666469971338
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,16383,0.01762666677435239
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,16383,0.01758933315674464
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,32767,0.03410666684309641
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,32767,0.01953599974513054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,65535,0.035530666510264076
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,65535,0.05249066650867462
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,131071,0.08703466256459554
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,1,0.011146667102972666
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,131071,0.053344001372655235
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,1,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,3,0.009370666618148485
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,3,0.01118933285276095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,7,0.009775999933481216
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,7,0.011114666859308878
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,15,0.011317333827416102
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,15,0.009461333354314169
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,31,0.010970667004585266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,31,0.011434666812419891
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,63,0.009514666472872099
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,63,0.01146666705608368
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,127,0.010437333335479101
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,127,0.012042666474978128
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,255,0.01091733326514562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,255,0.010037333394090334
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,511,0.011205332974592844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,511,0.011285333583752314
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,1023,0.010826667149861654
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,2047,0.01340266689658165
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,1023,0.01108266661564509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,2047,0.013082666943470636
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,4095,0.015285332997639975
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,4095,0.01393066719174385
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,8191,0.016501333564519882
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,8191,0.01533866673707962
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,16383,0.03196800003449122
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,16383,0.017829333742459614
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,32767,0.03309333324432373
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,32767,0.050399998823801674
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,65535,0.08468799789746602
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,65535,0.05054399867852529
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,131071,0.15491732954978943
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,131071,0.08603200316429138
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,1,0.01972266659140587
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,1,0.017664000391960144
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,3,0.019760000209013622
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,3,0.01762666677435239
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,15,0.019461333751678467
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,7,0.01969066634774208
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,7,0.017866666118303936
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,15,0.017375999440749485
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,31,0.01923199991385142
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,31,0.017701332767804463
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,63,0.017344000438849132
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,63,0.019440000255902607
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,127,0.019978666057189304
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,127,0.017509333789348602
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,255,0.029461334149042766
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,255,0.018160000443458557
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,511,0.048250665267308555
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,511,0.03160000095764796
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,1023,0.0827946662902832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,1023,0.05054933329423269
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,1,0.029370665550231934
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,2047,0.15517866611480713
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,2047,0.08436266581217448
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,1,0.02762666592995326
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,3,0.029333333174387615
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,3,0.027727998793125153
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,7,0.03001066545645396
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,15,0.029466666281223297
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,15,0.027722666660944622
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,31,0.02985599885384242
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,7,0.028421332438786823
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,31,0.027482666075229645
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,63,0.029893333713213604
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,63,0.02738133321205775
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,127,0.03631466627120972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,127,0.027450665831565857
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,255,0.05072533090909322
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,255,0.03580799947182337
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,511,0.05198933184146881
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,511,0.08527466654777527
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,1023,0.15436800320943198
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,1023,0.08738133311271667
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,2047,0.2940799991289775
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,2047,0.15808000167210898
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,1,0.029685333371162415
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,1,0.02752533306678136
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,3,0.029466666281223297
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,3,0.02829333394765854
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,7,0.029765332738558452
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,7,0.027552001178264618
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,15,0.029440000653266907
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,31,0.02977066735426585
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,31,0.027482666075229645
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,15,0.027658666173617046
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,63,0.029680001238981884
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,63,0.02775466690460841
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,127,0.03555733213822047
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,127,0.02773866554101308
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,255,0.050010666251182556
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,255,0.03568533311287562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,511,0.08726400136947632
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,511,0.05251200000445048
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,1023,0.15666133165359497
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,1023,0.0881866713364919
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,1,0.04433066646258036
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,3,0.052746668457984924
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,1,0.053583999474843345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,7,0.053247998158137
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,3,0.04410133262475332
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,7,0.04374399781227112
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,15,0.05262400209903717
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,31,0.05307200054327647
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,31,0.043893332282702126
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,15,0.04418666660785675
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,63,0.052671998739242554
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,63,0.04422933359940847
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,127,0.049498667319615684
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,255,0.08771199981371562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,127,0.058176000912984215
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,255,0.05819199979305267
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,511,0.09035733342170715
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,511,0.15775466958681741
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,1,0.0102613332370917
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,3,0.010357333347201347
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,1,0.011685332904259363
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,3,0.011343999455372492
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,7,0.011648000528415045
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,7,0.010874666273593903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,15,0.01108266661564509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,15,0.010874666273593903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,31,0.011050666371981302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,31,0.011541333049535751
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,63,0.010842667271693548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,63,0.011557333171367645
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,127,0.01101333275437355
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,127,0.011445333560307821
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,255,0.013264000415802002
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,255,0.011391999820868174
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,511,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,1023,0.2981226642926534
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,511,0.011183999478816986
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,1023,0.011168000598748526
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,2047,0.01320533330241839
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,2047,0.013450667262077332
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,4095,0.01498666654030482
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,4095,0.015077333897352219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,8191,0.017055999487638474
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,8191,0.015392000476519266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,16383,0.03505599995454153
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,16383,0.018005333840847015
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,32767,0.05013866722583771
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,1023,0.011002667248249054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,32767,0.033887999753157295
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,65535,0.08532266815503438
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,65535,0.050901333491007485
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,131071,0.15667733550071716
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,1,0.010650667051474253
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,131071,0.08738133311271667
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,3,0.010837333897749582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,1,0.011114666859308878
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,3,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,7,0.010863999525705973
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,7,0.010960000256697336
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,15,0.010805333654085795
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,15,0.009039999917149544
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,31,0.010922666639089584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,31,0.010794666906197866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,63,0.009141333401203156
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,63,0.011039999624093374
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,127,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,127,0.00973866693675518
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,255,0.01116266722480456
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,255,0.010768000036478043
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,511,0.01101333275437355
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,511,0.011445333560307821
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,1023,0.011317333827416102
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,1023,0.011002667248249054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,2047,0.012261333564917246
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,4095,0.013850666582584381
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,2047,0.013125333935022354
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,8191,0.029365333418051403
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,4095,0.013343999783198038
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,8191,0.016341333587964375
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,16383,0.0480373352766037
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,16383,0.031258667508761086
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,32767,0.08343467116355896
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,32767,0.048570667703946434
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,65535,0.1539413332939148
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,65535,0.08430400490760803
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,131071,0.2954026659329732
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,1023,0.1609280010064443
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,131071,0.15733333428700766
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,1,0.010266666611035665
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,1,0.01145600030819575
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,3,0.011535999675591787
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,7,0.01128000020980835
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,7,0.009408000235756239
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,15,0.009882666791478792
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,15,0.011264000087976456
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,31,0.009594666461149851
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,31,0.011461333682139715
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,3,0.009775999933481216
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,63,0.009141333401203156
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,63,0.011338666081428528
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,127,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,127,0.009850666547815004
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,255,0.011183999478816986
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,255,0.01166933278242747
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,511,0.011231999844312668
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,511,0.011685332904259363
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,1023,0.011498666057984034
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,2047,0.013114667187134424
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,2047,0.011829332758982977
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,1023,0.012437333663304647
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,4095,0.01553600033124288
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,4095,0.01320533330241839
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,8191,0.030282666285832722
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,8191,0.01791999985774358
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,16383,0.031370667119820915
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,16383,0.048298666874567665
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,32767,0.08385599652926128
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,32767,0.05029866596062978
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,65535,0.15559466679890951
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,65535,0.08529067039489746
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,1,0.00892800030608972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,1,0.011061333119869232
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,3,0.009018666421373686
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,3,0.011322667201360067
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,7,0.008901333436369896
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,131071,0.29707733790079754
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,7,0.011450666934251785
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,15,0.009018666421373686
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,31,0.008943999807039896
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,15,0.00902399979531765
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,31,0.009525333220760027
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,131071,0.15794133146603903
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,63,0.008997333546479544
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,63,0.009445333232482275
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,127,0.009375999992092451
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,127,0.0107893335322539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,255,0.008853333070874214
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,255,0.009072000160813332
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,511,0.011450666934251785
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,1023,0.009173333023985228
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,511,0.011253333340088526
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,1023,0.010954666882753372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,2047,0.010954666882753372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,2047,0.009882666791478792
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,4095,0.011429333438475927
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,4095,0.011125333607196808
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,8191,0.015642666568358738
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,8191,0.014885333677132925
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,16383,0.01932799940307935
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,16383,0.01941866676012675
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,32767,0.02770666778087616
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,32767,0.02659733345111211
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,65535,0.03086400032043457
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,65535,0.029733332494894665
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,131071,0.031504000226656594
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,131071,0.034202667574087776
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,1,0.009919999788204828
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,1,0.009568000212311745
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,3,0.008976000050703684
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,3,0.010170666500926018
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,7,0.009712000067035357
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,7,0.009642666826645533
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,15,0.008965333302815756
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,15,0.01121066634853681
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,31,0.009306666751702627
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,31,0.009349333122372627
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,63,0.008896000062425932
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,63,0.011402666568756104
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,127,0.010773333410422007
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,255,0.009109333157539368
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,127,0.009103999783595404
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,255,0.01121066634853681
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,511,0.010634666929642359
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,511,0.011301333705584208
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,1023,0.010384000216921171
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,1023,0.01109333336353302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,2047,0.011018666128317514
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,2047,0.011007999380429586
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,4095,0.011066666493813196
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,8191,0.015077333897352219
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,4095,0.011744000017642975
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,8191,0.015344000111023584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,16383,0.019280000279347103
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,16383,0.01904533306757609
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,32767,0.021498667697111767
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,32767,0.02345066765944163
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,65535,0.02346133440732956
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,65535,0.023141334454218548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,131071,0.025487999121348064
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,131071,0.04317333300908407
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,1,0.01099733387430509
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,1,0.01098666712641716
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,3,0.011109333485364914
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,3,0.01097600037852923
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,7,0.010389333590865135
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,7,0.011274666835864386
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,15,0.010944000134865442
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,15,0.01109333336353302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,31,0.01080000028014183
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,31,0.010570666442314783
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,63,0.01007466639081637
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,63,0.011952000359694162
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,127,0.011514666179815928
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,127,0.011002667248249054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,255,0.011178666104873022
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,255,0.0103946669648091
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,511,0.011354666203260422
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,511,0.011413333316644033
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,1023,0.013210666676362356
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,1023,0.012106666962305704
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,2047,0.015669333438078564
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,2047,0.013568000247081121
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,4095,0.03050133337577184
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,8191,0.04853333532810211
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,4095,0.015301333119471868
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,8191,0.03161599983771642
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,16383,0.08489599823951721
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,16383,0.05008000135421753
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,32767,0.1564586659272512
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,32767,0.0851146678129832
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,65535,0.2978399991989136
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,65535,0.15568533539772034
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,1,0.010890666395425797
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,1,0.01090666651725769
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,3,0.011077333241701126
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,3,0.010949333508809408
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,7,0.01128000020980835
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,131071,0.29737067222595215
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,131071,0.5812693436940511
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,7,0.010837333897749582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,15,0.011130666981140772
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,31,0.010794666906197866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,63,0.009653333574533463
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,31,0.009088000282645226
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,63,0.010847999403874079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,15,0.009301333377758661
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,127,0.00980266680320104
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,127,0.008997333546479544
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,255,0.010608000059922537
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,255,0.009525333220760027
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,511,0.01129066695769628
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,511,0.010842667271693548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,1023,0.010837333897749582
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,1023,0.010869332899649939
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,2047,0.01101333275437355
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,2047,0.010751999914646149
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,4095,0.011247999966144562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,4095,0.011098666737476984
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,8191,0.015717333803574245
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,8191,0.015008000036080679
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,16383,0.017562666287024815
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,16383,0.01743999992807706
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,32767,0.01913600042462349
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,32767,0.021301334102948506
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,65535,0.037130666275819145
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,65535,0.02145066608985265
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,131071,0.054005334774653115
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,131071,0.03830400109291077
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,1,0.011178666104873022
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,1,0.011120000233252844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,3,0.011002667248249054
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,3,0.011402666568756104
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,7,0.011114666859308878
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,7,0.010672000547250112
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,15,0.010954666882753372
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,15,0.011503999431928
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,63,0.011039999624093374
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,31,0.011231999844312668
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,31,0.011077333241701126
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,63,0.01138666644692421
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,127,0.010858666151762009
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,127,0.011178666104873022
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,255,0.01102399950226148
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,255,0.01228800043463707
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,511,0.011029332876205444
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,1023,0.013477332890033722
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,511,0.011365332951148352
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,1023,0.013269333789745966
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,2047,0.029189333319664
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,2047,0.015317333241303762
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,4095,0.04836266736189524
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,8191,0.08392533659934998
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,4095,0.031141333281993866
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,8191,0.04892266790072123
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,16383,0.1562399963537852
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,16383,0.08481066425641377
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,32767,0.2975733280181885
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,32767,0.15593066811561584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,65535,0.5806933244069418
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,65535,0.29900266726811725
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,1,0.013274667163689932
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,3,0.012757333616415659
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,7,0.013450667262077332
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,3,0.01190399999419848
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,1,0.012154666086037954
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,7,0.01110400011142095
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,15,0.012981332838535309
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,15,0.013376000026861826
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,31,0.011887999872366587
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,31,0.011087999989589056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,63,0.013280000537633896
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,63,0.01341333364446958
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,127,0.012618667135636011
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,127,0.013445333888133367
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,255,0.013386666774749756
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,255,0.01333333303531011
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,511,0.013647999614477158
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,511,0.013658666362365087
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,1023,0.03065066784620285
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,1023,0.017008000363906223
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,2047,0.0484799991051356
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,2047,0.03162666658560435
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,4095,0.08338133494059245
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,8191,0.15566399693489075
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,4095,0.05028266708056132
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,8191,0.08660800258318584
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,16383,0.15800533692042032
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,16383,0.2975626587867737
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,1,0.015397333850463232
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,1,0.013264000415802002
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,3,0.015237333873907724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,3,0.015360000232855478
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,7,0.015392000476519266
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,7,0.013301332791646322
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,15,0.015146666516860327
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,15,0.015205333630243937
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,31,0.013440000514189402
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,31,0.015589332828919092
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,63,0.015146666516860327
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,63,0.013482666263977686
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,127,0.01534933348496755
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,127,0.013845333208640417
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,255,0.01544533297419548
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,255,0.013477332890033722
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,511,0.02998399982849757
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,1023,0.04890666902065277
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,511,0.01738133281469345
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,1023,0.03164800008138021
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,2047,0.050474668542544045
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,2047,0.08541867136955261
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,4095,0.1590559979279836
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,4095,0.08658666412035625
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,1,0.010234666367371878
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,3,0.011461333682139715
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,3,0.011264000087976456
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,7,0.01173866664369901
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,8191,0.3035680055618286
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,7,0.011472000430027643
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,1,0.011178666104873022
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,15,0.010213333492477735
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,31,0.010751999914646149
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,15,0.009088000282645226
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,8191,0.15753066539764404
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,63,0.009695999945203463
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,31,0.010437333335479101
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,63,0.009072000160813332
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,127,0.010901333143313726
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,127,0.010677333921194077
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,255,0.00914666677514712
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,255,0.009759999811649323
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,511,0.011247999966144562
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,511,0.011007999380429586
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,1023,0.011120000233252844
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,1023,0.01138666644692421
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,2047,0.011226666470368704
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,2047,0.011584000041087469
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,4095,0.01313599944114685
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,8191,0.015189333508412043
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,4095,0.01333333303531011
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,16383,0.01703466723362605
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,8191,0.015509333461523056
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,16383,0.016890666137139004
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,32767,0.018005333840847015
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,32767,0.03389866650104523
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,65535,0.05258133510748545
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,65535,0.03549866626660029
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,131071,0.08815999825795491
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,131071,0.05351999898751577
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,1,0.01794133335351944
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,3,0.01913600042462349
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,1,0.01970133309563001
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,3,0.01759999990463257
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,7,0.01720533271630605
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,15,0.01735466718673706
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,7,0.019424000134070713
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,15,0.019679999599854153
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,31,0.01916266605257988
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,31,0.017765333255132038
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,63,0.019658666104078293
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,127,0.01923199991385142
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,63,0.01764800027012825
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,127,0.017509333789348602
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,255,0.030767999589443207
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,255,0.017125333348910015
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,511,0.04788800080617269
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,511,0.0316746657093366
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,1023,0.08493866523106892
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,1023,0.04896000027656555
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,2047,0.1540000041325887
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,2047,0.08449066678682964
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,4095,0.29658667246500653
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,4095,0.15636266271273294
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,1,0.029845332105954487
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,1,0.02731200059254964
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,3,0.02741333345572154
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,7,0.02734400083621343
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,7,0.029487999776999157
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,3,0.03188266605138779
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,15,0.029696000119050343
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,15,0.027562665442625683
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,31,0.029994666576385498
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,31,0.02736533433198929
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,63,0.02941333254178365
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,63,0.027621333797772724
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,127,0.03623466690381368
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,127,0.028117333849271137
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,255,0.05026666820049286
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,255,0.03610666592915853
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,511,0.08566932876904805
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,511,0.052485331892967224
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,1023,0.1567253371079763
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,1,0.009599999835093817
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,1,0.011146667102972666
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,3,0.009178666397929192
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,3,0.013151999562978745
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,7,0.009589333087205887
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,7,0.010847999403874079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,15,0.009189333145817121
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,15,0.009178666397929192
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,31,0.009354666496316591
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,31,0.009061333412925402
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,63,0.010453333457310995
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,63,0.01101333275437355
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,127,0.011007999380429586
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,1023,0.08813333511352539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,127,0.011178666104873022
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,255,0.00897066667675972
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,511,0.01109333336353302
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,511,0.0107893335322539
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,1023,0.010762666662534079
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,255,0.01116266722480456
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,1023,0.011130666981140772
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,2047,0.013349333157142004
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,2047,0.013066666821638743
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,4095,0.015285332997639975
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,4095,0.01322666679819425
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,8191,0.016202667107184727
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,8191,0.01522133375207583
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,16383,0.0327360009153684
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,16383,0.017594666530688603
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,32767,0.03322133421897888
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,32767,0.05028266708056132
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,65535,0.0517546683549881
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,65535,0.08627733588218689
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,131071,0.08904000123341878
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,131071,0.15625600020090738
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,2047,0.29652265707651776
VLLM,0.14.0,NVIDIA GB200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,2047,0.1590559979279836
