framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,1,0.011488000551859537
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,31,0.011450666934251785
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,3,0.011077333241701126
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,31,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,63,0.010949333508809408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,127,0.01108266661564509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,127,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,255,0.011168000598748526
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,1,0.011429333438475927
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,255,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,63,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,511,0.010885333021481832
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,1023,0.013445333888133367
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,2047,0.02939733366171519
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,2047,0.017157333592573803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,4095,0.04816000163555145
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,511,0.01210133358836174
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,4095,0.0314026673634847
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,1,0.011130666981140772
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,3,0.011247999966144562
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,3,0.011087999989589056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,3,0.011274666835864386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,7,0.011120000233252844
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,15,0.011098666737476984
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,7,0.011168000598748526
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,7,0.011087999989589056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,15,0.011109333485364914
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,1023,0.013461332768201828
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,1,0.010954666882753372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,31,0.011141333729028702
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,63,0.011034666250149408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,63,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,127,0.010928000013033548
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,255,0.011221333096424738
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,511,0.013061333447694778
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,7,0.011141333729028702
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,1023,0.029418667157491047
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,15,0.011071999867757162
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,31,0.011157333850860596
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,2047,0.031471999982992806
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,127,0.011237333218256632
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,4095,0.08469333251317342
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,1,0.00921066664159298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,1,0.010362666721145311
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,3,0.008912000184257826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,3,0.009258666386206945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,511,0.013002666334311167
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,7,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,1023,0.015930666277805965
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,15,0.009343999748428663
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,31,0.009578666960199675
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,31,0.009029333169261614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,63,0.01002133327225844
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,15,0.011114666859308878
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,255,0.010981333752473196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,127,0.010863999525705973
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,255,0.009008000294367472
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,255,0.010079999764760336
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,7,0.009045333291093508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,511,0.011071999867757162
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,1023,0.013151999562978745
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,15,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,2047,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,2047,0.04798933366934458
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,4095,0.010933333386977514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,4095,0.010949333508809408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,63,0.009279999881982803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,4095,0.050255998969078064
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,8191,0.012058666596810022
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,16383,0.01727466657757759
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,16383,0.017130666722853977
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,1,0.008879999940594038
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,1,0.00922133338948091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,3,0.008938666433095932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,7,0.008874666566650072
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,7,0.009045333291093508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,1023,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,15,0.008933333059151968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,2047,0.011007999380429586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,31,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,31,0.008912000184257826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,8191,0.013104000439246496
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,63,0.008842666943868002
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,127,0.009061333412925402
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,127,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,255,0.00892800030608972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,511,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,3,0.009194666519761086
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,1023,0.010874666273593903
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,1023,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,15,0.009114666531483332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,2047,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,4095,0.010938666760921478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,63,0.009472000102202097
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,127,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,8191,0.01458666721979777
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,255,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,16383,0.016127999871969223
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,511,0.010741333166758219
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,1,0.009306666751702627
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,3,0.009343999748428663
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,3,0.009093333035707474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,2047,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,7,0.00919999989370505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,7,0.010304000228643417
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,15,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,15,0.010309333602587381
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,8191,0.013173333058754602
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,31,0.009701333319147428
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,16383,0.01717866708834966
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,63,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,511,0.010954666882753372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,127,0.009178666397929192
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,255,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,511,0.011071999867757162
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,511,0.011125333607196808
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,4095,0.01121066634853681
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,1023,0.01110400011142095
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,1023,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,2047,0.011034666250149408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,31,0.00980266680320104
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,4095,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,4095,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,8191,0.015333333363135656
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,8191,0.01524266724785169
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,16383,0.01714133347074191
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,1,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,1,0.008912000184257826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,3,0.009088000282645226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,3,0.009205333267649015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,7,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,7,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,15,0.009136000027259191
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,15,0.008949333180983862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,2047,0.010954666882753372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,31,0.009077333534757296
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,63,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,1,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,16383,0.017125333348910015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,127,0.008933333059151968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,127,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,255,0.009402666861812273
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,511,0.010933333386977514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,511,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,1023,0.015103999525308609
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,1023,0.01090666651725769
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,2047,0.011152000476916632
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,2047,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,31,0.009338666374484697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,4095,0.013114667187134424
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,63,0.010357333347201347
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,255,0.009130666653315226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,255,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,16383,0.017312000195185345
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,1,0.013082666943470636
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,3,0.013194666554530462
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,4095,0.012938667088747025
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,3,0.01310933381319046
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,8191,0.017103999853134155
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,7,0.013045333325862885
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,16383,0.03145066648721695
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,15,0.013210666676362356
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,15,0.013050666699806849
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,31,0.012960000584522883
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,1,0.013056000073750814
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,63,0.013173333058754602
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,63,0.012901333471139273
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,127,0.013023999830087027
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,127,0.013125333935022354
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,255,0.013232000172138214
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,255,0.013077333569526672
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,63,0.009029333169261614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,511,0.015210667004187902
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,511,0.014991999914248785
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,127,0.008896000062425932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,1023,0.031023999055226643
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,2047,0.04884799818197886
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,2047,0.032842665910720825
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,1,0.013674666484196981
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,1,0.013301332791646322
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,3,0.014384000251690546
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,7,0.013066666821638743
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,15,0.014463999619086584
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,8191,0.015087999403476715
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,15,0.01301866645614306
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,31,0.014021333307027817
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,31,0.01303999995191892
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,1023,0.017269333203633625
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,31,0.01302933320403099
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,63,0.014752000570297241
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,127,0.01516266663869222
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,3,0.01314666618903478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,255,0.015018666783968607
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,255,0.012928000340859095
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,511,0.03073066721359889
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,7,0.015146666516860327
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,1023,0.049466664592425026
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,1023,0.031397332747777305
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,1,0.00922133338948091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,1,0.009194666519761086
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,7,0.013178666432698568
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,3,0.009205333267649015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,3,0.010357333347201347
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,7,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,63,0.013034666577974955
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,15,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,127,0.013157332936922709
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,31,0.009258666386206945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,511,0.01704000060757001
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,63,0.00922133338948091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,2047,0.04976533353328705
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,2047,0.0853706697622935
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,127,0.00892800030608972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,255,0.009045333291093508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,7,0.010277333358923594
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,511,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,511,0.011055999745925268
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,1023,0.011087999989589056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,2047,0.011109333485364914
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,63,0.008885333314538002
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,4095,0.01504533365368843
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,4095,0.013210666676362356
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,8191,0.017093333105246227
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,8191,0.01573333392540614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,127,0.009994666402538618
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,16383,0.033333333830038704
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,255,0.008912000184257826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,1,0.009077333534757296
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,1,0.00980266680320104
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,3,0.008890666688481966
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,3,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,1023,0.011007999380429586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,7,0.00914666677514712
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,7,0.00921066664159298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,15,0.009039999917149544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,31,0.00890666681031386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,2047,0.011055999745925268
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,63,0.009056000038981438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,63,0.009952000031868616
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,127,0.009103999783595404
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,255,0.00892800030608972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,16383,0.019178666174411774
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,511,0.010954666882753372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,511,0.010960000256697336
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,1023,0.011237333218256632
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,1023,0.01099733387430509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,2047,0.011952000359694162
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,2047,0.011328000575304031
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,31,0.009141333401203156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,4095,0.013023999830087027
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,8191,0.029733332494894665
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,8191,0.015146666516860327
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,16383,0.047824000318845115
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,16383,0.03133333226044973
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,1,0.01721599946419398
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,1,0.01509333277742068
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,15,0.009050666665037474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,4095,0.01402666668097178
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,3,0.017055999487638474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,7,0.018906666586796444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,127,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,7,0.014959999670584997
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,15,0.01704000060757001
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,15,0.015040000279744467
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,31,0.01714133347074191
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,31,0.01505600040157636
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,15,0.009173333023985228
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,63,0.015024000157912573
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,3,0.015050667027632395
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,255,0.017210666090250015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,255,0.00903466654320558
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,511,0.031397332747777305
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,511,0.019226666539907455
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,63,0.01714666684468587
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,1023,0.05022400120894114
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,31,0.011157333850860596
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,127,0.01706133286158244
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,127,0.015013333410024643
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,3,0.017173333714405697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,255,0.01532799998919169
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,7,0.019296000401178997
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,7,0.017349333812793095
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,15,0.019199999670187633
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,15,0.017658667018016178
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,31,0.017152000218629837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,1023,0.033471999069054924
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,63,0.01933866615096728
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,1,0.01930133377512296
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,127,0.019258666783571243
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,127,0.017322666943073273
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,3,0.019178666174411774
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,255,0.017184000462293625
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,511,0.04921066761016846
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,31,0.019258666783571243
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,511,0.03183466692765554
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,1023,0.05130666494369507
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,63,0.01727466657757759
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,1,0.025392000873883564
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,1,0.021242665747801464
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,3,0.025290665527184803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,255,0.031199999153614044
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,7,0.021242665747801464
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,15,0.024298667907714844
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,1023,0.0848479966322581
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,15,0.02130666623512904
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,31,0.02518400053183238
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,31,0.021216000119845074
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,63,0.024586667617162068
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,3,0.021242665747801464
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,63,0.021290667355060577
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,127,0.02526933451493581
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,7,0.024735999604066212
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,127,0.021274665991465252
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,255,0.031530665854612984
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,1,0.03229333211978277
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,1,0.02608533451954524
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,3,0.02978666623433431
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,1,0.01735466718673706
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,3,0.027215999861558277
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,7,0.030853333572546642
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,15,0.029477333029111225
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,15,0.025306666890780132
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,31,0.029850666721661884
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,63,0.03148799886306127
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,255,0.020970667401949566
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,63,0.02640533447265625
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,127,0.03382399926582972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,127,0.025621332228183746
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,255,0.04994666576385498
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,255,0.03323200096686681
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,31,0.02738133321205775
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,1,0.033402666449546814
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,3,0.0417546679576238
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,3,0.0323786661028862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,7,0.04163199911514918
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,7,0.03249066571394602
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,7,0.025744001070658367
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,15,0.04162666698296865
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,15,0.032261334359645844
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,1,0.04163199911514918
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,31,0.03179733455181122
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,63,0.04170133173465729
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,63,0.03156800071398417
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,127,0.033530667424201965
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,127,0.04403733213742574
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,1,0.04578666885693868
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,3,0.05551466842492422
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,31,0.04164266586303711
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,3,0.04589866598447164
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,7,0.0543146679798762
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,7,0.04603200157483419
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,15,0.05461333195368449
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,15,0.04588800172011057
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,1,0.05541866521040598
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,31,0.05533333122730255
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,31,0.04599999884764353
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,1,0.009296000003814697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,3,0.010682666053374609
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,3,0.010250666489203772
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,7,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,7,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,15,0.00938666673998038
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,63,0.04574933151404063
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,63,0.054341331124305725
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,127,0.04799999793370565
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,63,0.00978133330742518
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,127,0.011231999844312668
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,127,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,255,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,255,0.009317333499590555
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,511,0.011173332730929056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,1023,0.01108266661564509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,31,0.009434666484594345
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,1023,0.011205332974592844
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,127,0.05806399881839752
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,63,0.010357333347201347
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,2047,0.013114667187134424
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,4095,0.014127999544143677
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,8191,0.03051200012365977
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,4095,0.015125333021084467
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,8191,0.017157333592573803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,16383,0.048197334011395775
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,16383,0.03173866619666418
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,1,0.009296000003814697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,1,0.009365333244204521
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,3,0.00915733352303505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,3,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,31,0.009706666693091393
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,7,0.009461333354314169
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,2047,0.012938667088747025
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,15,0.009072000160813332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,31,0.009136000027259191
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,31,0.00926399976015091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,63,0.008949333180983862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,127,0.008922666932145754
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,511,0.011114666859308878
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,255,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,255,0.00914666677514712
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,511,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,15,0.01090666651725769
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,511,0.011114666859308878
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,1023,0.011247999966144562
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,15,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,1023,0.011066666493813196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,2047,0.013056000073750814
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,4095,0.02959999938805898
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,4095,0.01505600040157636
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,127,0.010458666831254959
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,8191,0.03138133386770884
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,16383,0.08473066488901775
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,7,0.009114666531483332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,16383,0.04981866478919983
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,2047,0.013343999783198038
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,63,0.009173333023985228
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,1,0.0100426667680343
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,8191,0.04797866443792979
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,1,0.06018133461475372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,3,0.05801066756248474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,float16,3,0.07819733520348866
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,7,0.05804799993832906
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,float16,15,0.07717866698900859
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,float16,1,0.0767626663049062
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,float16,7,0.07707733412583669
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,15,0.05819199979305267
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,31,0.05832533538341522
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,float16,63,0.07761600116888683
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,63,0.05835199852784475
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,float16,1,0.09858133395512898
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,1,0.08532266815503438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,float16,31,0.07855999966462453
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,float16,3,0.09967466195424397
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,3,0.08516266942024231
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,float16,7,0.09930666287740071
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,7,0.08598400155703227
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,float16,15,0.09977066516876221
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,float16,31,0.09905067086219788
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,31,0.08507200082143147
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,float16,63,0.09946667154630025
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,15,0.0846560001373291
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,63,0.0846720039844513
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,float16,1,0.14498133460680643
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,1,0.10734400153160095
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,float16,3,0.14549332857131958
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,3,0.10838933785756429
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,float16,7,0.14660267035166422
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,7,0.10789866248766582
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,float16,15,0.14418133099873862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,15,0.10824533303578694
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,float16,31,0.14414933323860168
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,float16,1,0.18481600284576416
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,31,0.10784000158309937
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,1,0.15878400206565857
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,float16,3,0.18449066082636514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,3,0.15869333346684775
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,1,0.009072000160813332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,float16,7,0.1843199928601583
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,7,0.15802666544914246
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,3,0.009999999776482582
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,3,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,7,0.011183999478816986
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,15,0.01007466639081637
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,15,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,31,0.009839999799927076
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,31,0.010816000401973724
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,63,0.009632000078757605
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,63,0.011066666493813196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,127,0.010053333515922228
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,float16,15,0.1842986742655436
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,255,0.009813333551088968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,255,0.010965333630641302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,511,0.01126933346192042
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,7,0.010992000500361124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,511,0.010938666760921478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,1023,0.012970666090647379
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,15,0.1567359964052836
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,2047,0.015077333897352219
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,2047,0.013258667041858038
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,4095,0.030762667457262676
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,float16,31,0.18435200055440268
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,8191,0.048437332113583885
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,127,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,1,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,3,0.011039999624093374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,1,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,7,0.009226666763424873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,31,0.15863466262817383
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,15,0.009370666618148485
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,1023,0.012928000340859095
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,31,0.009237333511312803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,4095,0.017162666966517765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,8191,0.03148266673088074
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,63,0.01198400060335795
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,1,0.01156266654531161
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,127,0.010928000013033548
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,3,0.009226666763424873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,255,0.011114666859308878
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,7,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,511,0.01089599976936976
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,1023,0.013045333325862885
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,1023,0.01201066623131434
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,31,0.010714666297038397
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,2047,0.02942933390537898
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,63,0.009674666449427605
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,4095,0.047877331574757896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,4095,0.030426666140556335
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,8191,0.04827199876308441
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,255,0.010410666465759277
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,1,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,3,0.010725333044926325
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,3,0.01099733387430509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,2047,0.015061333775520325
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,7,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,127,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,15,0.011087999989589056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,15,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,31,0.010832000523805618
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,31,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,511,0.011253333340088526
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,1,0.010922666639089584
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,127,0.01119999960064888
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,255,0.010928000013033548
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,15,0.009301333377758661
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,511,0.011429333438475927
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,511,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,7,0.010965333630641302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,1023,0.01301866645614306
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,2047,0.029322666426499683
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,63,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,2047,0.01716800034046173
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,4095,0.03109866629044215
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,127,0.01128000020980835
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,1,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,3,0.01089599976936976
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,3,0.01090666651725769
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,7,0.011215999722480774
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,7,0.011007999380429586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,15,0.010954666882753372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,15,0.010992000500361124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,63,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,31,0.011034666250149408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,63,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,255,0.011114666859308878
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,127,0.01097600037852923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,1023,0.013434667140245438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,255,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,255,0.010965333630641302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,511,0.01414399966597557
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,511,0.013343999783198038
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,4095,0.04772266745567322
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,1023,0.029834667841593426
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,1,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,2047,0.04841066896915436
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,2047,0.03121600051720937
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,63,0.011141333729028702
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,4095,0.08472533027331035
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,127,0.011071999867757162
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,3,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,3,0.009045333291093508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,7,0.008912000184257826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,31,0.010965333630641302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,15,0.009253333633144697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,15,0.009359999870260557
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,31,0.009583999713261923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,63,0.009226666763424873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,4095,0.04951466619968414
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,1,0.00933333362142245
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,63,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,127,0.010026666646202406
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,8191,0.08362133304278056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,255,0.010026666646202406
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,511,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,511,0.010933333386977514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,7,0.009098666409651438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,1023,0.011114666859308878
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,31,0.009578666960199675
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,2047,0.011226666470368704
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,4095,0.011274666835864386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,4095,0.011173332730929056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,8191,0.012885333349307379
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,1,0.009712000067035357
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,16383,0.01714133347074191
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,127,0.008938666433095932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,255,0.009141333401203156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,1,0.008933333059151968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,3,0.009226666763424873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,7,0.008837333569924036
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,1023,0.015109332899252573
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,2047,0.011055999745925268
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,15,0.008933333059151968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,8191,0.012096000214417776
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,31,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,31,0.00903466654320558
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,16383,0.01708799973130226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,63,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,1,0.009189333145817121
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,127,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,127,0.009418666362762451
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,3,0.009242666885256767
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,255,0.00898133342464765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,511,0.01099733387430509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,1023,0.011349332829316458
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,511,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,1023,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,7,0.009269333134094873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,2047,0.010938666760921478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,2047,0.01091733326514562
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,4095,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,8191,0.01350933313369751
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,8191,0.013376000026861826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,16383,0.017136000096797943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,63,0.009418666362762451
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,1,0.00903466654320558
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,1,0.009162666896979014
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,255,0.008816000074148178
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,3,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,7,0.009919999788204828
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,1023,0.011002667248249054
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,15,0.008890666688481966
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,15,0.009098666409651438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,31,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,31,0.009306666751702627
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,63,0.009237333511312803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,127,0.009039999917149544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,127,0.010735999792814255
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,3,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,255,0.010015999898314476
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,511,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,511,0.011098666737476984
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,1023,0.010992000500361124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,15,0.009061333412925402
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,1023,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,2047,0.011098666737476984
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,2047,0.011071999867757162
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,4095,0.011039999624093374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,8191,0.01523200049996376
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,63,0.008890666688481966
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,16383,0.01754666616519292
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,16383,0.017045332739750545
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,1,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,255,0.009962666779756546
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,7,0.009039999917149544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,3,0.00956266683836778
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,7,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,7,0.009253333633144697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,15,0.00915733352303505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,4095,0.011231999844312668
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,15,0.009050666665037474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,31,0.009008000294367472
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,63,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,16383,0.01613866661985715
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,127,0.008922666932145754
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,127,0.009232000137368837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,255,0.008890666688481966
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,3,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,511,0.009685333197315535
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,1023,0.01099733387430509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,4095,0.010890666395425797
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,31,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,1023,0.010805333654085795
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,2047,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,2047,0.010885333021481832
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,4095,0.01303999995191892
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,4095,0.01328533391157786
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,8191,0.015247999380032221
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,1,0.008863999818762144
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,16383,0.03183999905983607
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,16383,0.017114666601022083
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,1,0.012970666090647379
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,511,0.011039999624093374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,3,0.01314666618903478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,3,0.01313599944114685
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,7,0.013034666577974955
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,15,0.01314666618903478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,15,0.012335999558369318
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,63,0.009306666751702627
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,31,0.012960000584522883
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,63,0.013104000439246496
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,255,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,63,0.012703999876976013
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,127,0.01310933381319046
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,1,0.012975999464591345
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,127,0.012991999586423239
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,255,0.013104000439246496
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,7,0.013151999562978745
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,511,0.014981333166360855
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,511,0.014864000181357065
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,8191,0.015109332899252573
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,1023,0.029733332494894665
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,2047,0.04884799818197886
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,2047,0.03156800071398417
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,1,0.015061333775520325
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,1,0.013072000195582708
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,3,0.013141332815090815
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,7,0.015146666516860327
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,7,0.01302933320403099
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,15,0.01421333352724711
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,15,0.01302933320403099
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,31,0.014159999787807465
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,63,0.015050667027632395
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,1023,0.01722666621208191
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,63,0.012986666212479273
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,127,0.014896000425020853
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,8191,0.015664000064134598
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,127,0.013151999562978745
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,3,0.014463999619086584
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,255,0.015087999403476715
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,255,0.012986666212479273
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,511,0.017050666113694508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,1023,0.04824000100294749
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,31,0.01312000056107839
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,1023,0.0321066677570343
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,2047,0.05020266771316528
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,1,0.009258666386206945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,3,0.009183999771873156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,3,0.009589333087205887
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,7,0.008922666932145754
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,7,0.009973333527644476
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,255,0.013056000073750814
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,15,0.008890666688481966
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,31,0.009237333511312803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,63,0.009119999905427298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,63,0.009077333534757296
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,1,0.009061333412925402
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,127,0.010751999914646149
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,31,0.013066666821638743
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,255,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,255,0.00914666677514712
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,511,0.01184533288081487
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,1023,0.010965333630641302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,31,0.009253333633144697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,2047,0.01099733387430509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,4095,0.01381333296497663
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,2047,0.08470400174458821
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,8191,0.017077332983414333
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,8191,0.016538667182127636
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,1023,0.01129066695769628
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,16383,0.019226666539907455
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,511,0.02978666623433431
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,2047,0.010965333630641302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,1,0.009349333122372627
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,3,0.00884799969693025
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,3,0.009183999771873156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,7,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,7,0.009642666826645533
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,15,0.009242666885256767
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,511,0.011157333850860596
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,15,0.01002133327225844
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,31,0.00890666681031386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,63,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,63,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,1,0.009050666665037474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,127,0.008949333180983862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,127,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,255,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,255,0.009189333145817121
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,511,0.011087999989589056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,1023,0.010933333386977514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,1023,0.011152000476916632
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,15,0.009317333499590555
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,16383,0.033413333197434746
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,4095,0.01504533365368843
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,4095,0.013072000195582708
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,8191,0.029872000217437744
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,8191,0.015344000111023584
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,16383,0.048170665899912514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,127,0.009242666885256767
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,511,0.010911999891201654
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,1,0.01711999997496605
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,1,0.015200000256299973
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,2047,0.011109333485364914
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,31,0.00884799969693025
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,7,0.017071999609470367
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,7,0.015194666882356008
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,15,0.017184000462293625
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,15,0.015029333531856537
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,16383,0.031397332747777305
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,63,0.01708799973130226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,63,0.015072000523408255
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,2047,0.014501333236694336
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,3,0.01505600040157636
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,127,0.0170666662355264
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,127,0.015008000036080679
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,255,0.017237332959969837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,255,0.015061333775520325
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,511,0.031471999982992806
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,4095,0.014149333039919535
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,1023,0.049925332268079124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,1023,0.032858667274316154
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,1,0.017210666090250015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,3,0.017152000218629837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,3,0.01926400015751521
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,511,0.019098666807015736
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,1,0.019173332800467808
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,15,0.01741333305835724
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,7,0.017173333714405697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,31,0.019472000499566395
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,31,0.017173333714405697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,63,0.01738133281469345
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,127,0.01923199991385142
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,127,0.017018667111794155
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,7,0.01937066639463107
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,255,0.030154667794704437
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,31,0.014949332922697067
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,255,0.017184000462293625
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,63,0.019445333629846573
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,511,0.0495413343111674
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,511,0.03197866678237915
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,1023,0.08539733290672302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,1023,0.050944000482559204
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,31,0.01706133286158244
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,1,0.02515200028816859
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,3,0.02532266577084859
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,15,0.01929066702723503
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,1,0.019946667055288952
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,7,0.02426133304834366
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,15,0.02470933397610982
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,15,0.01937599976857503
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,31,0.025226667523384094
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,31,0.019765333582957584
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,3,0.01722666621208191
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,63,0.02532800038655599
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,3,0.0195573332409064
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,7,0.01989866668979327
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,255,0.021210665504137676
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,1,0.025648000339667004
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,3,0.02957333376010259
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,3,0.027349332968393963
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,7,0.030794667700926464
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,63,0.019637333850065868
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,7,0.025407999753952026
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,127,0.020080000162124634
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,255,0.0314026673634847
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,15,0.03134933362404505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,1,0.02977599948644638
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,15,0.026005332668622334
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,31,0.025685332715511322
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,63,0.02997333308060964
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,63,0.026474667092164356
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,127,0.026362667481104534
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,127,0.025279998779296875
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,255,0.03181866556406021
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,1,0.041663999358812966
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,1,0.03186666717131933
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,31,0.02958400050799052
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,3,0.04181866844495138
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,127,0.033717334270477295
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,7,0.04355733096599579
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,7,0.032186667124430336
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,255,0.05012799799442291
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,15,0.04167999823888143
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,31,0.042064001162846885
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,31,0.03182933231194814
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,3,0.032469332218170166
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,63,0.04173333446184794
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,63,0.03212266663710276
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,127,0.044826666514078774
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,127,0.033146666983763375
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,1,0.054618666569391884
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,1,0.045893331368764244
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,3,0.05459199845790863
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,3,0.04601066807905833
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,7,0.055231998364130654
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,7,0.04637333254019419
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,15,0.04584533472855886
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,31,0.054144000013669334
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,15,0.03305066625277201
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,63,0.05439466734727224
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,3,0.009381333366036415
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,127,0.057674666245778404
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,3,0.010469333579142889
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,15,0.05416533350944519
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,31,0.04571733375390371
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,7,0.009141333401203156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,15,0.009226666763424873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,31,0.009045333291093508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,31,0.009962666779756546
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,63,0.04620266457398733
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,127,0.009904000287254652
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,127,0.04785599807898203
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,127,0.010981333752473196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,7,0.010415999839703241
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,255,0.009088000282645226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,511,0.010853332777818045
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,15,0.010890666395425797
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,63,0.009205333267649015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,1,0.011509332805871964
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,2047,0.013194666554530462
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,4095,0.015189333508412043
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,4095,0.013381333400805792
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,255,0.010058666889866194
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,8191,0.02940800040960312
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,8191,0.01721599946419398
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,16383,0.04822933177153269
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,16383,0.031871999303499855
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,1023,0.0120319997270902
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,1,0.009269333134094873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,63,0.010351999973257383
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,3,0.00956266683836778
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,3,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,7,0.00892800030608972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,7,0.009205333267649015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,15,0.00926399976015091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,15,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,31,0.009455999980370203
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,31,0.00943999985853831
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,63,0.009482666850090027
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,63,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,127,0.009583999713261923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,1,0.009375999992092451
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,2047,0.012954667210578918
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,255,0.01033599985142549
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,255,0.009935999910036722
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,511,0.010922666639089584
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,1023,0.011781333635250727
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,1023,0.011226666470368704
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,2047,0.012981332838535309
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,4095,0.015109332899252573
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,511,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,127,0.009722666814923286
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,8191,0.03145066648721695
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,16383,0.0495306650797526
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,16383,0.08406399687131245
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,511,0.010901333143313726
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,2047,0.014138666292031607
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,4095,0.03443733354409536
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,1,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,8191,0.04790933430194855
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,float16,1,0.07885333398977916
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,1023,0.011055999745925268
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,7,0.05780800183614095
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,float16,7,0.07851733267307281
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,float16,3,0.0791733314593633
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,1,0.056048000852266945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,3,0.05831466615200043
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,15,0.05795733133951823
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,float16,31,0.07861333092053731
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,float16,15,0.0779306689898173
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,31,0.05602133274078369
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,float16,63,0.07853866616884868
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,float16,1,0.09805867075920105
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,63,0.05809600154558817
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,float16,3,0.09914132952690125
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,1,0.08502933382987976
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,3,0.08582400282224019
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,float16,7,0.09929066896438599
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,7,0.08547199765841167
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,15,0.08531733353932698
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,float16,31,0.09900266925493877
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,31,0.08474133412043254
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,float16,63,0.09944533308347066
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,63,0.08482133348782857
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,float16,15,0.09902933239936829
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,float16,1,0.14572266737620035
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,1,0.10711999734242757
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,float16,3,0.14589333534240723
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,3,0.10778666536013286
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,float16,7,0.1478506624698639
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,7,0.10691199700037639
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,float16,15,0.14718400438626608
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,15,0.1072106659412384
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,float16,31,0.1453120013078054
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,31,0.10691199700037639
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,float16,1,0.18496533234914145
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,1,0.158351997534434
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,float16,3,0.18443200985590616
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,1,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,1,0.009648000200589498
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,3,0.009226666763424873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,float16,7,0.18502400318781534
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,3,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,7,0.15834133823712668
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,7,0.009925333162148794
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,15,0.009269333134094873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,15,0.009269333134094873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,31,0.009125333279371262
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,31,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,63,0.00983466642598311
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,63,0.011087999989589056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,127,0.0100853331387043
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,3,0.15847999850908914
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,255,0.00898133342464765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,255,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,511,0.011077333241701126
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,511,0.011050666371981302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,1023,0.013077333569526672
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,1023,0.011711999773979187
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,2047,0.015029333531856537
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,float16,31,0.185205340385437
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,4095,0.030293333033720653
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,31,0.1583199997742971
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,4095,0.017194667210181553
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,127,0.011178666104873022
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,1,0.00921066664159298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,8191,0.04850133260091146
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,1,0.010346666599313417
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,8191,0.032101333141326904
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,7,0.010026666646202406
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,3,0.00914666677514712
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,7,0.009152000149091085
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,15,0.15839999914169312
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,15,0.01009599988659223
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,31,0.009482666850090027
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,31,0.01027199998497963
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,2047,0.014080000420411428
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,63,0.009205333267649015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,63,0.010330666477481524
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,float16,15,0.18433600664138794
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,255,0.01116266722480456
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,255,0.011039999624093374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,511,0.011194666226704916
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,7,0.009258666386206945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,511,0.010981333752473196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,1023,0.013130666067202887
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,2047,0.02942933390537898
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,4095,0.04789333542188009
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,127,0.009237333511312803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,4095,0.030421334008375805
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,8191,0.048453330993652344
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,3,0.009599999835093817
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,1,0.010960000256697336
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,15,0.01099733387430509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,1023,0.011535999675591787
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,3,0.01108266661564509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,2047,0.014874666929244995
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,7,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,15,0.010949333508809408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,15,0.011034666250149408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,31,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,63,0.01099733387430509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,63,0.010922666639089584
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,127,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,127,0.010954666882753372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,1,0.011087999989589056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,255,0.01097600037852923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,7,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,511,0.011039999624093374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,1023,0.013477332890033722
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,127,0.011077333241701126
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,2047,0.01714666684468587
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,8191,0.08357333143552144
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,4095,0.031343999008337654
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,1,0.011285333583752314
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,511,0.011322667201360067
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,1,0.01090666651725769
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,3,0.011087999989589056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,3,0.010992000500361124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,2047,0.029440000653266907
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,7,0.01091733326514562
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,15,0.011098666737476984
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,255,0.011039999624093374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,31,0.011237333218256632
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,31,0.011136000355084738
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,63,0.01118933285276095
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,1023,0.013088000317414602
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,31,0.01099733387430509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,127,0.011264000087976456
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,7,0.01110400011142095
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,255,0.01121066634853681
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,255,0.011125333607196808
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,511,0.01303999995191892
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,3,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,1023,0.029685333371162415
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,1023,0.015423999478419622
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,2047,0.03136533250411352
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,4095,0.08418132861455281
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,63,0.011066666493813196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,4095,0.050255998969078064
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,1,0.010069333637754122
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,1,0.009296000003814697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,3,0.010069333637754122
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,7,0.00972800018886725
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,511,0.013194666554530462
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,15,0.012448000411192576
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,15,0.009248000259200731
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,31,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,31,0.008922666932145754
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,63,0.009173333023985228
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,63,0.00922133338948091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,127,0.011338666081428528
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,127,0.009397333487868309
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,3,0.009658666948477427
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,255,0.008896000062425932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,511,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,4095,0.04788800080617269
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,1023,0.01118933285276095
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,1023,0.011130666981140772
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,2047,0.011253333340088526
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,4095,0.010960000256697336
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,2047,0.047872001926104225
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,4095,0.01097600037852923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,8191,0.01302933320403099
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,8191,0.011962667107582092
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,255,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,16383,0.017136000096797943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,7,0.010794666906197866
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,1,0.008933333059151968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,511,0.011034666250149408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,3,0.009269333134094873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,15,0.009061333412925402
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,7,0.009050666665037474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,7,0.009599999835093817
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,2047,0.010960000256697336
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,15,0.009946666657924652
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,31,0.009088000282645226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,31,0.009610666582981745
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,127,0.008933333059151968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,63,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,63,0.009269333134094873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,127,0.008821333448092142
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,255,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,255,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,1,0.009141333401203156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,511,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,3,0.009242666885256767
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,1023,0.011114666859308878
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,1023,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,2047,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,2047,0.011258666714032492
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,4095,0.012122667084137598
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,4095,0.010938666760921478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,8191,0.013365333278973898
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,16383,0.01743999992807706
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,1,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,127,0.00898133342464765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,1,0.009503999724984169
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,3,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,7,0.00927466650803884
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,511,0.01090666651725769
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,7,0.009008000294367472
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,15,0.00916800027092298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,15,0.01073066641887029
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,31,0.009194666519761086
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,63,0.009306666751702627
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,127,0.009919999788204828
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,8191,0.015103999525308609
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,16383,0.017231999586025875
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,127,0.009482666850090027
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,255,0.009290666629870733
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,255,0.010079999764760336
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,511,0.01089599976936976
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,1023,0.011130666981140772
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,1023,0.01128000020980835
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,2047,0.010992000500361124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,2047,0.011071999867757162
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,31,0.00890666681031386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,15,0.00892800030608972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,8191,0.015210667004187902
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,16383,0.017221332838137943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,16383,0.017125333348910015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,16383,0.015674666812022526
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,511,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,1,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,3,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,3,0.009077333534757296
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,4095,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,7,0.00890666681031386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,4095,0.01118933285276095
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,15,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,15,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,63,0.010640000303586325
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,63,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,63,0.009125333279371262
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,127,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,1,0.008885333314538002
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,3,0.009194666519761086
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,255,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,255,0.009162666896979014
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,511,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,511,0.011087999989589056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,7,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,1023,0.010928000013033548
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,8191,0.015226667126019796
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,4095,0.012970666090647379
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,8191,0.016496000190575916
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,8191,0.015189333508412043
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,16383,0.03148266673088074
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,16383,0.017450666675964992
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,1023,0.011296000331640244
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,2047,0.011050666371981302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,1,0.013125333935022354
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,4095,0.012938667088747025
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,3,0.012874666601419449
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,127,0.009248000259200731
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,7,0.012965332716703415
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,15,0.0129120002190272
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,15,0.012928000340859095
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,31,0.012981332838535309
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,31,0.012965332716703415
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,2047,0.01116266722480456
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,31,0.008890666688481966
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,63,0.013023999830087027
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,3,0.013104000439246496
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,127,0.013023999830087027
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,127,0.012719999998807907
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,255,0.01312000056107839
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,7,0.013194666554530462
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,511,0.014442666123310724
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,1023,0.030378667016824085
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,1023,0.017269333203633625
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,63,0.012991999586423239
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,31,0.008890666688481966
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,1,0.012975999464591345
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,255,0.013477332890033722
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,511,0.01524266724785169
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,3,0.013359999905029932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,7,0.01422400027513504
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,7,0.01302933320403099
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,2047,0.031744000812371574
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,1,0.012986666212479273
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,15,0.012975999464591345
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,31,0.014725333700577417
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,31,0.01328533391157786
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,63,0.014085333794355392
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,3,0.014282666146755219
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,127,0.015024000157912573
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,127,0.01332266628742218
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,2047,0.048565333088239036
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,255,0.015082667271296183
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,255,0.013306666165590286
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,511,0.03105599929889043
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,15,0.01404800017674764
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,1023,0.048810665806134544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,1,0.009557333464423815
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,2047,0.08612266182899475
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,2047,0.04994133114814758
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,3,0.00922133338948091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,3,0.00919999989370505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,7,0.009232000137368837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,7,0.00922133338948091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,15,0.009039999917149544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,31,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,63,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,1,0.014783999572197596
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,511,0.017050666113694508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,63,0.010415999839703241
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,127,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,255,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,255,0.009328000247478485
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,127,0.010037333394090334
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,511,0.010960000256697336
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,511,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,1023,0.011296000331640244
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,2047,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,15,0.01033599985142549
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,2047,0.011071999867757162
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,4095,0.013754667093356451
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,31,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,8191,0.01718933383623759
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,16383,0.03170666595300039
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,1023,0.03181866556406021
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,1,0.008901333436369896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,1,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,1,0.01089599976936976
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,3,0.009242666885256767
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,63,0.013077333569526672
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,1023,0.012752000242471695
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,7,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,15,0.009434666484594345
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,15,0.008874666566650072
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,31,0.009381333366036415
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,4095,0.013893333574136099
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,63,0.00898133342464765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,8191,0.015386667102575302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,63,0.009061333412925402
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,127,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,127,0.00933333362142245
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,255,0.010128000130256018
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,255,0.009338666374484697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,511,0.010869332899649939
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,511,0.010954666882753372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,1023,0.011002667248249054
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,2047,0.012272000312805176
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,7,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,4095,0.013093333691358566
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,8191,0.030048000315825146
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,16383,0.047781333327293396
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,16383,0.031130666534105938
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,1023,0.010992000500361124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,1,0.0170666662355264
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,2047,0.011328000575304031
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,4095,0.015066667149464289
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,1,0.015216000378131866
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,3,0.016773333152135212
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,31,0.008832000195980072
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,7,0.01720000058412552
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,7,0.015141333142916361
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,15,0.01708799973130226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,15,0.015050667027632395
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,16383,0.019141333798567455
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,31,0.01613866661985715
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,31,0.015066667149464289
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,63,0.01714666684468587
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,63,0.01515199989080429
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,127,0.015130666395028433
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,255,0.017194667210181553
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,255,0.015066667149464289
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,8191,0.015647999942302704
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,511,0.019178666174411774
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,1023,0.04996266464392344
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,1023,0.03297066688537598
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,1,0.017152000218629837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,3,0.019280000279347103
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,3,0.01710933322707812
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,7,0.019194666296243668
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,511,0.03172266731659571
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,7,0.01720533271630605
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,15,0.019167999426523846
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,15,0.018245333184798557
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,31,0.019253333409627277
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,3,0.009349333122372627
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,63,0.01937066639463107
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,63,0.017429333180189133
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,3,0.015114666273196539
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,127,0.017173333714405697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,255,0.030410667260487873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,1,0.019445333629846573
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,255,0.01714666684468587
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,511,0.04920533299446106
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,1023,0.08481599887212117
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,511,0.03143999973932902
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,1,0.02518933266401291
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,127,0.01912533367673556
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,3,0.025034666061401367
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,1,0.019695999721686046
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,3,0.020154666155576706
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,7,0.023941333095232647
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,7,0.020224000016848247
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,15,0.02350933353106181
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,1023,0.050527999798456825
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,31,0.025274666647116344
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,31,0.019920000185569126
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,63,0.024842667082945507
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,127,0.025397333006064098
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,127,0.021194666624069214
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,255,0.03139200061559677
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,255,0.02092266579469045
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,127,0.017258666455745697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,1,0.029482667644818623
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,3,0.030117332935333252
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,63,0.019653332730134327
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,3,0.027376001079877216
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,7,0.030928000807762146
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,15,0.029557332396507263
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,15,0.025813333690166473
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,31,0.03014933317899704
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,31,0.01709866647919019
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,1,0.027424000203609467
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,63,0.030458666384220123
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,63,0.025360000630219776
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,7,0.02550400048494339
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,127,0.026357332865397137
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,255,0.04978133241335551
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,255,0.03336533407370249
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,15,0.019637333850065868
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,1,0.04204800228277842
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,1,0.03147733211517334
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,3,0.041663999358812966
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,127,0.03339733431736628
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,3,0.03180799881617228
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,7,0.04178666571776072
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,15,0.04178133110205332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,31,0.02738133321205775
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,15,0.03145066648721695
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,31,0.04178666571776072
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,31,0.03288000077009201
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,63,0.03249066571394602
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,127,0.04373333354791006
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,1,0.0539626677831014
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,3,0.0544053316116333
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,3,0.045791998505592346
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,63,0.04164266586303711
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,7,0.05479466418425242
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,7,0.03152533372243246
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,127,0.03369600077470144
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,15,0.05485333502292633
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,15,0.04590400060017904
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,7,0.04554133117198944
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,31,0.04584533472855886
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,1,0.04589866598447164
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,1,0.01099733387430509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,3,0.009242666885256767
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,127,0.05598933498064677
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,127,0.04780800143877665
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,7,0.009237333511312803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,15,0.010015999898314476
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,15,0.009008000294367472
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,31,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,31,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,63,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,63,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,3,0.010458666831254959
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,127,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,7,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,31,0.05421866476535797
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,255,0.008949333180983862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,511,0.011098666737476984
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,511,0.010960000256697336
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,1023,0.011711999773979187
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,2047,0.013045333325862885
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,2047,0.013125333935022354
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,63,0.04555733501911163
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,4095,0.015135999768972397
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,1,0.009152000149091085
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,8191,0.03072533259789149
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,4095,0.013562666873137156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,8191,0.01709866647919019
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,16383,0.031856000423431396
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,255,0.009039999917149544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,1,0.00903466654320558
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,3,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,3,0.009328000247478485
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,7,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,15,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,63,0.05426133175690969
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,15,0.009242666885256767
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,16383,0.04867733518282572
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,31,0.009642666826645533
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,1,0.009253333633144697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,63,0.00984533317387104
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,127,0.009919999788204828
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,1023,0.010928000013033548
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,255,0.009269333134094873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,7,0.009077333534757296
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,511,0.010890666395425797
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,1023,0.011109333485364914
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,1023,0.011226666470368704
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,2047,0.013104000439246496
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,2047,0.013093333691358566
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,4095,0.029765332738558452
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,4095,0.015066667149464289
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,127,0.009125333279371262
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,8191,0.04795200129350027
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,8191,0.031370667119820915
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,255,0.009599999835093817
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,16383,0.04958933095137278
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,511,0.011034666250149408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,31,0.009082666908701261
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,63,0.009381333366036415
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,16383,0.0836853285630544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,1,0.056464001536369324
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,float16,1,0.07853866616884868
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,127,0.009242666885256767
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,float16,3,0.0786240001519521
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,3,0.05818133552869161
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,float16,7,0.0784853349129359
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,7,0.05614933371543884
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,float16,15,0.07852266728878021
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,15,0.056602666775385536
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,float16,31,0.07846400141716003
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,31,0.05740800003210703
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,float16,63,0.07875733574231465
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,63,0.05808533231417338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,float16,1,0.09875733653704326
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,1,0.08477333188056946
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,float16,3,0.09922666351000468
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,3,0.08483200271924336
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,float16,7,0.09934399525324504
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,7,0.08572799960772197
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,float16,15,0.09947199622790019
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,15,0.08682666222254436
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,float16,31,0.09809066851933797
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,31,0.08473599950472514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,float16,63,0.09924800197283427
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,63,0.08500799536705017
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,float16,1,0.14664533734321594
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,1,0.10693333546320598
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,float16,3,0.14723733067512512
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,3,0.10735467076301575
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,float16,7,0.14612799882888794
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,7,0.10730133454004924
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,float16,15,0.14682666460673013
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,15,0.10724266370137532
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,float16,31,0.14613333344459534
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,float16,1,0.1837706764539083
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,1,0.15848533312479654
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,float16,3,0.18430399894714355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,3,0.15819733341534933
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,31,0.10726400216420491
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,1,0.009306666751702627
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,1,0.009941333283980688
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,float16,7,0.18357332547505698
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,3,0.010911999891201654
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,7,0.1578879952430725
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,7,0.009653333574533463
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,7,0.01108266661564509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,15,0.010901333143313726
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,15,0.009701333319147428
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,31,0.010661333799362183
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,63,0.009872000043590864
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,63,0.010415999839703241
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,127,0.0099093330403169
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,127,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,255,0.009888000165422758
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,255,0.010869332899649939
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,511,0.011055999745925268
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,31,0.010928000013033548
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,1023,0.013072000195582708
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,1023,0.013082666943470636
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,2047,0.013290667285521826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,4095,0.03002133220434189
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,3,0.00960533320903778
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,float16,31,0.18336000045140585
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,511,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,8191,0.048485333720842995
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,8191,0.03141333411137263
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,float16,15,0.18399999539057413
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,31,0.1584053337574005
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,2047,0.015978666643301647
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,7,0.01027199998497963
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,3,0.009594666461149851
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,3,0.011002667248249054
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,7,0.010960000256697336
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,15,0.009632000078757605
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,15,0.010778666784365972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,4095,0.017231999586025875
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,31,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,63,0.010735999792814255
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,63,0.010965333630641302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,1,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,127,0.010768000036478043
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,1,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,255,0.010885333021481832
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,511,0.011226666470368704
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,1023,0.013002666334311167
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,15,0.15846932927767435
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,2047,0.02940800040960312
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,31,0.010597333312034607
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,127,0.010922666639089584
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,4095,0.030784000953038532
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,8191,0.08369599779446919
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,4095,0.047983999053637184
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,1,0.009754666437705358
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,3,0.009343999748428663
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,1,0.010869332899649939
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,1023,0.013967999567588171
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,7,0.009786666681369146
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,15,0.011066666493813196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,15,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,31,0.01097600037852923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,63,0.010874666273593903
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,255,0.010965333630641302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,63,0.01028266673286756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,511,0.011136000355084738
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,3,0.010687999427318573
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,255,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,2047,0.015114666273196539
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,511,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,31,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,1023,0.013013333082199097
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,1023,0.01322666679819425
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,8191,0.04810666541258494
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,127,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,4095,0.03138133386770884
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,4095,0.01706133286158244
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,7,0.0107893335322539
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,8191,0.03142933299144109
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,511,0.01097600037852923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,2047,0.015168000012636185
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,2047,0.01414399966597557
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,3,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,127,0.010661333799362183
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,7,0.00927466650803884
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,255,0.010687999427318573
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,15,0.009493333597977957
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,15,0.009061333412925402
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,1,0.009557333464423815
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,31,0.00926399976015091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,63,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,63,0.00916800027092298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,127,0.010938666760921478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,127,0.009072000160813332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,255,0.010602666685978571
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,511,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,511,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,7,0.009162666896979014
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,1023,0.012863999853531519
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,2047,0.029466666281223297
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,31,0.010640000303586325
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,4095,0.047882666190465294
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,1,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,4095,0.030389333764712017
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,8191,0.08361066381136577
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,255,0.010037333394090334
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,1,0.011205332974592844
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,3,0.011055999745925268
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,3,0.011055999745925268
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,1023,0.013130666067202887
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,7,0.011039999624093374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,8191,0.04985600213209788
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,7,0.011007999380429586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,15,0.011077333241701126
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,15,0.011157333850860596
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,63,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,3,0.011258666714032492
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,127,0.011322667201360067
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,1,0.01089599976936976
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,255,0.01138666644692421
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,255,0.011050666371981302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,511,0.013466666142145792
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,511,0.013125333935022354
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,2047,0.015066667149464289
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,1023,0.029487999776999157
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,63,0.011050666371981302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,8191,0.04987733562787374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,2047,0.04784533381462097
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,2047,0.031354665756225586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,127,0.011152000476916632
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,4095,0.08472533027331035
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,4095,0.0498986691236496
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,8191,0.15768000483512878
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,8191,0.08668800195058186
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,1,0.009685333197315535
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,3,0.009930666536092758
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,3,0.010319999729593595
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,31,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,15,0.010159999753038088
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,1023,0.015029333531856537
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,31,0.009658666948477427
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,31,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,63,0.009301333377758661
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,63,0.00922133338948091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,127,0.009535999968647957
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,1,0.009205333267649015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,511,0.011039999624093374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,511,0.011034666250149408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,7,0.008933333059151968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,31,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,1023,0.011087999989589056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,2047,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,4095,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,127,0.009189333145817121
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,4095,0.011130666981140772
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,255,0.008858666444818178
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,8191,0.012378666549921036
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,255,0.009183999771873156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,16383,0.01717866708834966
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,32767,0.019610666980346043
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,1,0.00914666677514712
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,15,0.009088000282645226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,3,0.009328000247478485
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,3,0.009162666896979014
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,7,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,8191,0.011674666156371435
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,7,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,16383,0.015173333386580149
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,15,0.008922666932145754
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,32767,0.020938667158285778
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,31,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,7,0.009957333405812582
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,63,0.009290666629870733
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,1023,0.011285333583752314
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,127,0.00898133342464765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,255,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,255,0.009162666896979014
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,511,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,1023,0.011050666371981302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,1023,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,15,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,2047,0.01116266722480456
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,31,0.008869333192706108
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,4095,0.01099733387430509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,63,0.009039999917149544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,8191,0.011370666325092316
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,1,0.00919999989370505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,16383,0.015114666273196539
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,127,0.009082666908701261
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,32767,0.019551999866962433
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,511,0.010992000500361124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,1,0.009066666786869368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,3,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,3,0.010010666524370512
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,2047,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,4095,0.011141333729028702
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,2047,0.01090666651725769
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,15,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,16383,0.015002666662136713
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,31,0.009029333169261614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,32767,0.01724799970785777
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,31,0.00922133338948091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,63,0.008922666932145754
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,63,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,127,0.009114666531483332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,127,0.009189333145817121
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,255,0.009573333586255709
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,255,0.008949333180983862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,511,0.010954666882753372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,1023,0.011098666737476984
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,7,0.009098666409651438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,1023,0.01090666651725769
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,8191,0.011066666493813196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,2047,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,4095,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,4095,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,8191,0.012997332960367203
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,1,0.009162666896979014
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,16383,0.015301333119471868
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,32767,0.019039999693632126
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,511,0.011034666250149408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,1,0.00921066664159298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,3,0.008885333314538002
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,2047,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,15,0.00892800030608972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,3,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,7,0.008922666932145754
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,7,0.009279999881982803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,16383,0.017301333447297413
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,15,0.010037333394090334
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,31,0.009050666665037474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,32767,0.03348266581694285
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,31,0.009232000137368837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,63,0.008922666932145754
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,127,0.009045333291093508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,7,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,255,0.009248000259200731
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,255,0.009285333255926767
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,511,0.011136000355084738
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,8191,0.014309333016475042
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,2047,0.010954666882753372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,15,0.00921066664159298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,2047,0.01110400011142095
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,4095,0.010949333508809408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,4095,0.011157333850860596
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,8191,0.013077333569526672
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,1,0.009237333511312803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,8191,0.011322667201360067
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,16383,0.017152000218629837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,32767,0.019386666516462963
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,16383,0.01706133286158244
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,32767,0.019189332922299702
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,1,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,511,0.010821333775917688
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,3,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,1023,0.011055999745925268
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,3,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,15,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,15,0.009226666763424873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,31,0.009056000038981438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,31,0.00916800027092298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,63,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,63,0.009082666908701261
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,127,0.009312000125646591
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,127,0.009103999783595404
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,1,0.009077333534757296
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,1023,0.012608000387748083
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,255,0.00891733355820179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,511,0.010960000256697336
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,511,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,1023,0.011050666371981302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,2047,0.010832000523805618
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,2047,0.010981333752473196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,4095,0.010949333508809408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,63,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,4095,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,8191,0.014111999422311783
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,127,0.009103999783595404
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,16383,0.016074666132529575
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,32767,0.03311999887228012
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,32767,0.019061333189407986
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,255,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,1,0.009136000027259191
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,3,0.009194666519761086
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,7,0.009237333511312803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,7,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,7,0.008879999940594038
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,15,0.00891733355820179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,31,0.00891733355820179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,16383,0.017152000218629837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,31,0.00903466654320558
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,63,0.008922666932145754
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,63,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,127,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,127,0.008922666932145754
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,255,0.008949333180983862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,1023,0.010837333897749582
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,255,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,511,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,511,0.010874666273593903
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,1023,0.011039999624093374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,2047,0.010890666395425797
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,2047,0.011141333729028702
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,8191,0.013077333569526672
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,4095,0.013088000317414602
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,4095,0.013306666165590286
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,8191,0.0164533331990242
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,8191,0.01509333277742068
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,16383,0.03169599920511246
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,7,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,32767,0.04983466863632202
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,32767,0.03341866781314214
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,1,0.010938666760921478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,15,0.009253333633144697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,3,0.010933333386977514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,7,0.010960000256697336
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,7,0.010784000158309937
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,15,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,1,0.009072000160813332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,15,0.010890666395425797
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,3,0.00903466654320558
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,31,0.010965333630641302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,1,0.011055999745925268
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,63,0.010922666639089584
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,127,0.011173332730929056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,255,0.011071999867757162
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,255,0.011071999867757162
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,16383,0.01714133347074191
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,511,0.011087999989589056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,63,0.010933333386977514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,511,0.011039999624093374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,1023,0.010992000500361124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,1023,0.013440000514189402
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,127,0.01109333336353302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,2047,0.029802667597929638
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,4095,0.04818666477998098
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,4095,0.031685332457224526
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,1,0.01137599969903628
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,1,0.011071999867757162
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,31,0.011050666371981302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,3,0.010938666760921478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,7,0.011301333705584208
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,3,0.010965333630641302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,1023,0.013013333082199097
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,31,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,2047,0.017018667111794155
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,63,0.01166933278242747
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,63,0.011173332730929056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,127,0.011445333560307821
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,127,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,3,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,255,0.010981333752473196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,511,0.013061333447694778
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,15,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,511,0.013056000073750814
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,1023,0.029792000850041706
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,31,0.011066666493813196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,2047,0.04800533254941305
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,2047,0.03148799886306127
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,4095,0.08477866649627686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,4095,0.05012266834576925
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,1,0.015135999768972397
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,1,0.013461332768201828
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,3,0.015130666395028433
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,255,0.011253333340088526
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,7,0.01479999969402949
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,15,0.015141333142916361
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,7,0.011098666737476984
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,31,0.013077333569526672
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,63,0.01481066644191742
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,63,0.013082666943470636
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,127,0.014767999450365702
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,3,0.012975999464591345
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,255,0.014490666488806406
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,255,0.013264000415802002
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,15,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,511,0.029445332785447437
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,1023,0.01516266663869222
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,511,0.01716800034046173
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,1023,0.04894400139649709
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,1023,0.03146666785081228
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,4095,0.15928000211715698
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,127,0.013023999830087027
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,7,0.012954667210578918
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,1,0.009008000294367472
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,3,0.009205333267649015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,31,0.015194666882356008
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,3,0.009279999881982803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,7,0.00926399976015091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,15,0.009248000259200731
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,15,0.009109333157539368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,31,0.008949333180983862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,31,0.00915733352303505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,63,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,63,0.009290666629870733
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,2047,0.05049600203831991
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,127,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,255,0.009242666885256767
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,255,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,511,0.010949333508809408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,15,0.012991999586423239
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,511,0.010885333021481832
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,1023,0.011039999624093374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,2047,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,7,0.009306666751702627
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,2047,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,4095,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,8191,0.01516266663869222
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,16383,0.01777600000301997
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,4095,0.0867786705493927
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,16383,0.017071999609470367
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,1,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,32767,0.020655999581019085
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,1023,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,3,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,4095,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,3,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,8191,0.015082667271296183
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,7,0.009189333145817121
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,7,0.009253333633144697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,15,0.009578666960199675
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,2047,0.08594133456548055
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,31,0.009130666653315226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,32767,0.033520000676314034
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,1,0.008869333192706108
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,1,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,127,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,127,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,255,0.009056000038981438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,255,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,511,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,511,0.010778666784365972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,1023,0.011034666250149408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,31,0.008912000184257826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,127,0.009392000113924345
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,2047,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,63,0.009039999917149544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,8191,0.016224000602960587
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,8191,0.015178666760524115
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,16383,0.03153600047032038
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,15,0.00914666677514712
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,16383,0.017258666455745697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,1023,0.010911999891201654
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,32767,0.04985066751639048
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,1,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,63,0.008816000074148178
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,1,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,3,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,7,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,15,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,15,0.009423999736706415
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,31,0.008832000195980072
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,63,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,63,0.009130666653315226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,32767,0.03347733368476232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,4095,0.013130666067202887
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,3,0.009093333035707474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,4095,0.01302933320403099
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,255,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,255,0.009306666751702627
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,511,0.010666667173306147
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,1023,0.011183999478816986
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,31,0.009066666786869368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,2047,0.013194666554530462
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,2047,0.011306667079528173
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,4095,0.015253332753976187
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,4095,0.01309866706530253
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,2047,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,8191,0.02995733420054118
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,8191,0.01568000018596649
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,16383,0.04794666667779287
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,16383,0.03148799886306127
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,32767,0.04952533543109894
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,511,0.011077333241701126
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,1,0.01309866706530253
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,1023,0.011007999380429586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,3,0.013002666334311167
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,3,0.013173333058754602
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,7,0.013002666334311167
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,7,0.012949333836634954
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,127,0.009642666826645533
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,32767,0.08411733309427898
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,31,0.01301866645614306
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,31,0.013050666699806849
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,1,0.013082666943470636
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,63,0.012960000584522883
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,127,0.012970666090647379
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,127,0.013104000439246496
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,255,0.013093333691358566
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,15,0.013157332936922709
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,15,0.012800000607967377
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,511,0.015317333241303762
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,7,0.008890666688481966
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,1023,0.03140799949566523
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,1023,0.01714133347074191
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,2047,0.04980800052483877
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,2047,0.032042667269706726
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,1,0.01443733274936676
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,255,0.012981332838535309
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,1,0.013157332936922709
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,3,0.014885333677132925
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,127,0.009189333145817121
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,7,0.014938666174809137
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,7,0.01331199953953425
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,511,0.014357333381970724
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,15,0.013104000439246496
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,31,0.01341333364446958
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,63,0.014671999961137772
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,63,0.013013333082199097
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,127,0.01339200014869372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,255,0.015125333021084467
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,255,0.013365333278973898
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,3,0.013072000195582708
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,511,0.017136000096797943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,31,0.014378666877746582
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,1023,0.049135997891426086
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,1023,0.03200000027815501
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,127,0.01479999969402949
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,2047,0.050885334610939026
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,1,0.019173332800467808
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,3,0.01703466723362605
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,3,0.019280000279347103
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,511,0.03073599934577942
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,7,0.01926400015751521
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,15,0.015130666395028433
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,7,0.017423999806245167
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,15,0.019141333798567455
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,15,0.017077332983414333
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,31,0.019205333044131596
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,31,0.01718933383623759
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,63,0.01912533367673556
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,63,0.01706133286158244
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,127,0.01740266631046931
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,1,0.017173333714405697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,255,0.029818666477998097
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,255,0.01720000058412552
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,63,0.013050666699806849
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,511,0.04818133513132731
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,1023,0.050986667474110924
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,127,0.01939733326435089
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,2047,0.15664533774058023
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,2047,0.08801066875457764
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,1,0.01728533332546552
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,1,0.01525866612792015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,3,0.01711999997496605
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,2047,0.08629866441090901
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,3,0.015103999525308609
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,7,0.01716800034046173
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,1023,0.0846560001373291
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,15,0.01721599946419398
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,15,0.015040000279744467
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,31,0.017050666113694508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,31,0.015018666783968607
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,63,0.01509333277742068
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,127,0.01721599946419398
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,127,0.015285332997639975
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,255,0.015061333775520325
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,511,0.031386665999889374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,511,0.019173332800467808
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,1,0.01921066641807556
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,1,0.017360000560681026
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,3,0.01952533299724261
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,7,0.019093333433071773
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,3,0.01730666682124138
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,7,0.01717866708834966
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,255,0.01718933383623759
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,15,0.01747200017174085
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,31,0.01924266666173935
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,31,0.01736533393462499
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,63,0.017375999440749485
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,127,0.019386666516462963
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,127,0.01708799973130226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,511,0.031445334355036415
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,255,0.029946667452653248
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,255,0.01725333308180173
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,511,0.032314665615558624
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,511,0.04980800052483877
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,1,0.029370665550231934
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,1,0.02534399926662445
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,7,0.015109332899252573
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,3,0.02937600016593933
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,3,0.02552533398071925
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,63,0.019285333653291065
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,15,0.030048000315825146
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,15,0.025301332275072735
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,31,0.029653333127498627
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,31,0.027514666318893433
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,63,0.03018666555484136
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,7,0.02961066613594691
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,127,0.03334933271010717
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,63,0.01711999997496605
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,127,0.025557334224383037
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,255,0.04955733319123586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,255,0.03163733333349228
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,15,0.019173332800467808
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,511,0.0864586631457011
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,63,0.025914666553338368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,7,0.02537599951028824
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,511,0.05131733417510986
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,1,0.02085866779088974
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,3,0.024304000039895374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,7,0.021216000119845074
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,15,0.023520000278949738
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,7,0.02533866713444392
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,1,0.02390933285156886
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,15,0.021061333517233532
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,31,0.021274665991465252
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,3,0.020576000213623047
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,63,0.023898666103680927
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,127,0.025386666258176167
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,127,0.02056533346573512
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,255,0.03183999905983607
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,255,0.02123733361562093
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,31,0.023290666441122692
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,1,0.031632001201311745
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,1,0.027610667049884796
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,3,0.03182933231194814
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,63,0.0200853335360686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,3,0.02758399893840154
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,7,0.03156800071398417
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,15,0.0315733328461647
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,15,0.027808000644048054
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,31,0.03141866624355316
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,31,0.027349332968393963
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,63,0.031504000226656594
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,127,0.03555733213822047
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,7,0.028981332977612812
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,255,0.035760000348091125
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,1,0.05479466418425242
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,63,0.02771199991305669
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,1,0.04593066871166229
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,127,0.027456000447273254
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,3,0.0539626677831014
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,255,0.051967998345692955
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,7,0.05438933273156484
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,3,0.045941332976023354
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,15,0.053957333167394005
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,15,0.04558399816354116
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,31,0.045754666129748024
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,63,0.05524266759554545
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,63,0.04474666714668274
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,1,0.009370666618148485
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,7,0.04450666904449463
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,127,0.05668266614278158
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,1,0.010650667051474253
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,7,0.009008000294367472
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,7,0.009925333162148794
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,255,0.08916800220807393
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,255,0.05684266487757365
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,31,0.00891733355820179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,3,0.009994666402538618
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,31,0.010309333602587381
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,63,0.010666667173306147
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,127,0.00983466642598311
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,15,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,127,0.011066666493813196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,255,0.009546666716535887
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,255,0.009056000038981438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,511,0.010901333143313726
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,511,0.01099733387430509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,1023,0.01099733387430509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,1023,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,2047,0.010938666760921478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,2047,0.010992000500361124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,4095,0.013077333569526672
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,8191,0.01714666684468587
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,127,0.04773333172003428
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,3,0.008869333192706108
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,16383,0.03286933402220408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,63,0.00903466654320558
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,32767,0.05077866713205973
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,32767,0.03381866713364919
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,1,0.009056000038981438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,1,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,15,0.010293333480755487
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,3,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,7,0.008949333180983862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,7,0.009258666386206945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,15,0.009082666908701261
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,31,0.009098666409651438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,31,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,63,0.008863999818762144
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,8191,0.015413332730531693
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,63,0.009338666374484697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,127,0.009136000027259191
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,255,0.009029333169261614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,255,0.009370666618148485
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,16383,0.01907733331123988
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,511,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,3,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,1023,0.011098666737476984
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,2047,0.013093333691358566
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,15,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,4095,0.01303999995191892
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,8191,0.029968000948429108
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,127,0.009194666519761086
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,8191,0.015082667271296183
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,16383,0.0480320006608963
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,511,0.010474666953086853
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,16383,0.03140799949566523
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,32767,0.08460799853006999
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,32767,0.04996266464392344
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,1023,0.010949333508809408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,3,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,4095,0.015119999647140503
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,7,0.009413333609700203
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,15,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,15,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,31,0.008901333436369896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,31,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,63,0.009077333534757296
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,2047,0.01128000020980835
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,127,0.009232000137368837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,127,0.009418666362762451
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,31,0.0540533314148585
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,255,0.008853333070874214
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,1,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,511,0.010965333630641302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,4095,0.01505600040157636
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,1023,0.01129066695769628
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,7,0.009098666409651438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,2047,0.013061333447694778
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,2047,0.013141332815090815
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,4095,0.029482667644818623
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,4095,0.015301333119471868
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,1,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,255,0.010687999427318573
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,8191,0.03105599929889043
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,63,0.009285333255926767
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,16383,0.08377599716186523
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,16383,0.04887466629346212
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,32767,0.08507200082143147
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,32767,0.1560373306274414
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,8191,0.047824000318845115
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,3,0.009861333295702934
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,1023,0.011221333096424738
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,1,0.04171200096607208
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,1,0.0323786661028862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,3,0.04176533222198486
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,511,0.010949333508809408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,3,0.03339199970165888
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,7,0.03178666780392329
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,15,0.04182933270931244
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,15,0.03216533362865448
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,31,0.041834667325019836
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,31,0.03219199925661087
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,7,0.04170133173465729
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,63,0.041834667325019836
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,63,0.03278933217128118
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,127,0.043653334180514015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,127,0.03358400116364161
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,1,0.05808533231417338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,1,0.050000001986821495
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,3,0.05801600217819214
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,7,0.058058664202690125
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,7,0.0498933345079422
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,15,0.057962665955225624
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,15,0.04994666576385498
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,31,0.05794133245944977
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,3,0.0498879998922348
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,31,0.04987733562787374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,63,0.05806399881839752
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,63,0.04978133241335551
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,127,0.0622026671965917
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,1,0.09934399525324504
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,127,0.05389333268006643
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,1,0.08427199721336365
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,3,0.0981280008951823
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,3,0.08481599887212117
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,7,0.09901866316795349
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,7,0.08494399984677632
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,15,0.08443733056386311
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,31,0.0986293355623881
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,15,0.09929600358009338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,31,0.08437866965929668
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,63,0.09909333785374959
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,63,0.08450667063395183
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,127,0.10318400462468465
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,float16,1,0.07645333309968312
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,1,0.05600533386071523
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,3,0.05599466462930044
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,float16,7,0.07686399916807811
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,7,0.055776000022888184
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,float16,3,0.07649066547552745
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,127,0.08458667000134786
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,15,0.056661332647005715
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,float16,15,0.07662400106589
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,float16,31,0.07617599765459697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,31,0.05599466462930044
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,float16,63,0.07624533275763194
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,63,0.05628266433874766
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,float16,1,0.1051680048306783
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,1,0.0939573347568512
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,float16,3,0.10589866836865743
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,3,0.09322667121887207
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,float16,7,0.10513599713643391
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,7,0.09338133533795674
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,float16,15,0.10587199529012044
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,15,0.09358933568000793
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,float16,31,0.10518399874369304
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,31,0.09347732861836751
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,float16,63,0.105103999376297
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,float16,1,0.18266665935516357
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,63,0.09291199843088786
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,1,0.15666666626930237
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,3,0.1567253371079763
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,float16,7,0.18420799573262533
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,7,0.1569706698258718
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,float16,3,0.18443200985590616
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,1,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,1,0.010314666976531347
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,3,0.010101333260536194
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,7,0.009173333023985228
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,float16,15,0.18440532684326172
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,7,0.009114666531483332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,15,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,15,0.009045333291093508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,31,0.008938666433095932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,31,0.010640000303586325
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,63,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,63,0.00996800015370051
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,3,0.009194666519761086
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,127,0.009045333291093508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,127,0.010965333630641302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,255,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,255,0.009637333452701569
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,511,0.011130666981140772
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,1023,0.011109333485364914
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,31,0.1569760044415792
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,float16,31,0.18408532937367758
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,63,0.15639467040697733
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,float16,63,0.18371200561523438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,2047,0.012981332838535309
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,2047,0.01267733300725619
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,511,0.010863999525705973
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,8191,0.030741333961486816
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,16383,0.04877866804599762
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,4095,0.013130666067202887
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,1,0.00892800030608972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,1023,0.011050666371981302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,3,0.00919999989370505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,3,0.009141333401203156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,7,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,7,0.009914666414260864
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,15,0.008912000184257826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,31,0.009109333157539368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,31,0.009258666386206945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,63,0.009349333122372627
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,8191,0.017263999829689663
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,63,0.0099093330403169
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,127,0.009061333412925402
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,15,0.1562879979610443
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,511,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,15,0.009290666629870733
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,511,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,1023,0.011834666132926941
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,1023,0.011253333340088526
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,2047,0.013157332936922709
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,2047,0.013082666943470636
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,4095,0.029504001140594482
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,255,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,8191,0.04820266862710317
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,8191,0.03155199935038885
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,16383,0.049914668003718056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,1,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,127,0.009061333412925402
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,1,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,16383,0.03204799940188726
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,3,0.009045333291093508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,4095,0.015141333142916361
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,7,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,15,0.009296000003814697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,15,0.009056000038981438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,31,0.00914666677514712
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,31,0.009637333452701569
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,63,0.009701333319147428
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,127,0.010314666976531347
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,255,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,3,0.009583999713261923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,127,0.009109333157539368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,255,0.010992000500361124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,7,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,511,0.010965333630641302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,1023,0.012938667088747025
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,1023,0.013066666821638743
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,2047,0.02961066613594691
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,2047,0.015040000279744467
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,63,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,4095,0.047482664386431374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,4095,0.03047466774781545
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,8191,0.08326933284600575
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,8191,0.04771733283996582
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,16383,0.1557866632938385
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,1,0.009850666547815004
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,16383,0.08399466673533122
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,3,0.009375999992092451
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,7,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,7,0.009269333134094873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,4095,0.014991999914248785
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,15,0.00919999989370505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,15,0.010954666882753372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,31,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,511,0.010965333630641302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,31,0.011002667248249054
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,63,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,127,0.00996800015370051
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,127,0.010911999891201654
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,255,0.010847999403874079
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,1,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,511,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,511,0.011034666250149408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,16383,0.08513599634170532
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,1023,0.012762666990359625
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,1,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,63,0.009050666665037474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,2047,0.014368000129858652
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,4095,0.030906667311986286
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,8191,0.04816000163555145
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,255,0.009941333283980688
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,8191,0.03291733314593633
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,1,0.009162666896979014
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,1,0.009173333023985228
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,3,0.010853332777818045
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,7,0.009658666948477427
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,255,0.009173333023985228
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,7,0.009232000137368837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,3,0.009872000043590864
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,1023,0.012058666596810022
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,31,0.009829333052039146
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,4095,0.017082666357358296
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,63,0.01099733387430509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,63,0.010890666395425797
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,127,0.010949333508809408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,3,0.009045333291093508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,255,0.009248000259200731
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,2047,0.01331199953953425
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,511,0.011136000355084738
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,31,0.010757333288590113
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,511,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,1023,0.013295999417702356
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,1023,0.013007999708255133
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,2047,0.01515199989080429
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,4095,0.04807466765244802
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,4095,0.030133334298928578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,8191,0.04845866560935974
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,15,0.009632000078757605
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,1,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,1,0.01110400011142095
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,3,0.011152000476916632
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,3,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,2047,0.029637334247430164
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,127,0.010389333590865135
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,7,0.01118933285276095
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,255,0.010869332899649939
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,31,0.011535999675591787
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,31,0.010911999891201654
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,8191,0.08402666449546814
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,63,0.011274666835864386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,63,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,127,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,127,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,255,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,255,0.011087999989589056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,511,0.013445333888133367
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,511,0.013077333569526672
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,1023,0.015184000134468079
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,2047,0.04818133513132731
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,2047,0.03139200061559677
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,4095,0.0843946635723114
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,4095,0.04993066688378652
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,8191,0.1574666698773702
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,1,0.009189333145817121
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,3,0.008853333070874214
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,3,0.00914666677514712
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,7,0.009338666374484697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,1023,0.02945599953333537
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,7,0.00898133342464765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,15,0.009472000102202097
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,15,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,31,0.009877333417534828
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,31,0.009888000165422758
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,63,0.008949333180983862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,63,0.00966933307548364
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,127,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,1,0.009098666409651438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,255,0.008901333436369896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,255,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,511,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,511,0.010981333752473196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,1023,0.011130666981140772
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,15,0.009306666751702627
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,2047,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,2047,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,4095,0.011125333607196808
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,4095,0.011130666981140772
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,8191,0.08612799644470215
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,8191,0.011221333096424738
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,16383,0.015173333386580149
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,16383,0.015365333606799444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,32767,0.020597333709398907
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,32767,0.019530666371186573
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,1,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,3,0.009392000113924345
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,7,0.01119999960064888
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,3,0.009242666885256767
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,15,0.010992000500361124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,7,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,15,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,8191,0.012970666090647379
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,15,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,31,0.009039999917149544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,15,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,63,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,63,0.009066666786869368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,127,0.008890666688481966
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,127,0.009178666397929192
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,1023,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,511,0.01121066634853681
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,1023,0.011226666470368704
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,1023,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,2047,0.01097600037852923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,31,0.008949333180983862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,2047,0.011125333607196808
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,4095,0.011055999745925268
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,4095,0.010863999525705973
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,255,0.009045333291093508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,1,0.009039999917149544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,511,0.01097600037852923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,8191,0.011002667248249054
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,16383,0.016069332758585613
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,7,0.009130666653315226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,32767,0.01916266605257988
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,1,0.00892800030608972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,3,0.008938666433095932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,127,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,3,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,7,0.008890666688481966
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,7,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,15,0.009039999917149544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,15,0.009045333291093508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,8191,0.01110400011142095
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,31,0.009093333035707474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,16383,0.015333333363135656
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,1,0.00919999989370505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,127,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,255,0.009402666861812273
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,255,0.009253333633144697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,511,0.010981333752473196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,511,0.01099733387430509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,1023,0.011055999745925268
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,1023,0.010981333752473196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,31,0.008949333180983862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,2047,0.0107893335322539
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,4095,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,4095,0.010954666882753372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,8191,0.015013333410024643
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,8191,0.013951999445756277
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,16383,0.017194667210181553
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,16383,0.016058667252461117
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,32767,0.03349333256483078
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,1,0.009370666618148485
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,1,0.009248000259200731
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,32767,0.018858666221300762
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,3,0.009253333633144697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,3,0.008885333314538002
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,7,0.009008000294367472
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,255,0.008858666444818178
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,15,0.009077333534757296
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,15,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,31,0.009066666786869368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,31,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,63,0.008901333436369896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,32767,0.019130667050679524
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,127,0.00890666681031386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,2047,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,255,0.00921066664159298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,255,0.009082666908701261
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,127,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,511,0.011125333607196808
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,7,0.008933333059151968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,1023,0.011178666104873022
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,2047,0.011007999380429586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,2047,0.010954666882753372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,63,0.008938666433095932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,4095,0.011050666371981302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,8191,0.013077333569526672
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,16383,0.017136000096797943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,127,0.009541333342591921
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,63,0.009045333291093508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,16383,0.01703466723362605
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,32767,0.019189332922299702
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,1,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,1,0.009119999905427298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,3,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,3,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,7,0.008896000062425932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,7,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,15,0.009056000038981438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,31,0.008912000184257826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,63,0.009237333511312803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,31,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,511,0.010496000448862711
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,63,0.009082666908701261
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,1023,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,127,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,255,0.00892800030608972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,255,0.008901333436369896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,4095,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,511,0.010928000013033548
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,511,0.010949333508809408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,1023,0.01099733387430509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,2047,0.010746666540702185
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,8191,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,2047,0.010960000256697336
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,4095,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,8191,0.013359999905029932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,8191,0.013130666067202887
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,32767,0.01921066641807556
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,16383,0.015429332852363586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,32767,0.03345066557327906
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,32767,0.019248000035683315
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,1,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,1,0.009008000294367472
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,3,0.00890666681031386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,15,0.009061333412925402
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,1023,0.011114666859308878
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,3,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,7,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,4095,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,7,0.008874666566650072
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,15,0.009066666786869368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,31,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,63,0.00898133342464765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,16383,0.017125333348910015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,63,0.009989333028594652
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,127,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,127,0.008938666433095932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,255,0.009072000160813332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,511,0.010858666151762009
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,511,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,2047,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,15,0.009183999771873156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,2047,0.011034666250149408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,31,0.00914666677514712
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,4095,0.01312000056107839
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,63,0.009056000038981438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,8191,0.015135999768972397
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,16383,0.03181866556406021
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,16383,0.017184000462293625
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,32767,0.033333333830038704
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,1023,0.01091733326514562
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,1,0.011007999380429586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,4095,0.012885333349307379
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,127,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,3,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,7,0.011007999380429586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,32767,0.05031999945640564
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,1023,0.011098666737476984
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,15,0.011125333607196808
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,3,0.011173332730929056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,8191,0.016127999871969223
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,63,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,63,0.01090666651725769
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,127,0.010981333752473196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,15,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,127,0.011002667248249054
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,31,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,255,0.011002667248249054
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,31,0.011152000476916632
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,511,0.01116266722480456
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,1023,0.01314666618903478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,2047,0.030058667063713074
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,7,0.011055999745925268
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,2047,0.01722666621208191
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,4095,0.049509331583976746
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,255,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,1,0.011034666250149408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,511,0.011215999722480774
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,1,0.01121066634853681
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,1023,0.01421333352724711
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,7,0.01099733387430509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,15,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,15,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,255,0.009248000259200731
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,31,0.010992000500361124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,63,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,63,0.010928000013033548
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,3,0.011183999478816986
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,127,0.010879999647537867
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,3,0.010960000256697336
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,255,0.012293333808581034
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,255,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,511,0.013978666315476099
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,31,0.01108266661564509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,1023,0.02934933453798294
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,1,0.011285333583752314
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,2047,0.03143999973932902
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,4095,0.08470400174458821
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,7,0.011130666981140772
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,4095,0.05054933329423269
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,1,0.01498666654030482
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,1,0.013151999562978745
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,511,0.01313599944114685
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,3,0.01303999995191892
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,4095,0.031744000812371574
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,1023,0.015141333142916361
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,7,0.01333333303531011
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,127,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,15,0.014373333503802618
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,31,0.01302933320403099
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,63,0.015098666151364645
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,127,0.014725333700577417
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,127,0.01321600005030632
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,3,0.014965333044528961
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,255,0.013845333208640417
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,255,0.012997332960367203
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,7,0.015077333897352219
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,511,0.03014933317899704
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,511,0.016885332763195038
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,2047,0.04809600114822388
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,1023,0.031514666974544525
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,31,0.014970666418472925
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,1,0.009290666629870733
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,2047,0.08496532837549846
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,1,0.009162666896979014
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,3,0.009408000235756239
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,3,0.009493333597977957
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,7,0.009183999771873156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,15,0.01309866706530253
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,1023,0.04786133269468943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,15,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,15,0.009258666386206945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,31,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,4095,0.1604746679464976
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,63,0.009232000137368837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,127,0.00891733355820179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,127,0.009290666629870733
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,255,0.009248000259200731
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,255,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,511,0.010858666151762009
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,7,0.009114666531483332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,511,0.011039999624093374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,1023,0.01129066695769628
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,2047,0.011264000087976456
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,63,0.01303999995191892
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,2047,0.01089599976936976
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,4095,0.011264000087976456
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,4095,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,63,0.00884799969693025
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,8191,0.015109332899252573
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,16383,0.01738133281469345
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,16383,0.01718933383623759
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,32767,0.021146667500336964
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,1,0.00892800030608972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,1,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,3,0.00903466654320558
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,3,0.00927466650803884
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,1023,0.011365332951148352
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,7,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,2047,0.05009600023428599
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,31,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,31,0.008912000184257826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,8191,0.015119999647140503
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,32767,0.034490667283535004
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,127,0.00890666681031386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,127,0.008938666433095932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,255,0.008933333059151968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,7,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,255,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,15,0.00890666681031386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,4095,0.08710400263468425
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,511,0.010837333897749582
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,1023,0.010858666151762009
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,2047,0.010885333021481832
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,2047,0.010949333508809408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,63,0.009029333169261614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,4095,0.013056000073750814
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,8191,0.015562667200962702
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,16383,0.031845333675543465
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,16383,0.017397332936525345
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,15,0.009343999748428663
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,32767,0.03347733368476232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,31,0.009029333169261614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,1023,0.01116266722480456
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,1,0.00955200009047985
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,4095,0.014837333311637243
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,8191,0.014917333920796713
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,7,0.00884799969693025
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,7,0.00922133338948091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,15,0.009183999771873156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,15,0.009205333267649015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,31,0.009125333279371262
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,32767,0.04993600149949392
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,31,0.009237333511312803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,63,0.00891733355820179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,63,0.009056000038981438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,127,0.00914666677514712
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,63,0.009583999713261923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,255,0.009056000038981438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,3,0.009029333169261614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,3,0.009637333452701569
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,511,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,1023,0.01097600037852923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,1023,0.01097600037852923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,2047,0.01110400011142095
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,4095,0.01509333277742068
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,4095,0.01312000056107839
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,511,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,8191,0.02938666691382726
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,8191,0.015077333897352219
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,16383,0.047914668917655945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,127,0.009786666681369146
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,32767,0.08304533362388611
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,32767,0.04844800134499868
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,255,0.009381333366036415
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,511,0.01121066634853681
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,1,0.01310933381319046
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,3,0.013023999830087027
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,2047,0.012954667210578918
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,7,0.013274667163689932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,7,0.013082666943470636
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,15,0.01302933320403099
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,31,0.01322666679819425
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,16383,0.031370667119820915
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,31,0.013093333691358566
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,63,0.013045333325862885
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,63,0.012821332861979803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,3,0.013210666676362356
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,255,0.013077333569526672
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,255,0.01310933381319046
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,511,0.01504533365368843
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,511,0.015018666783968607
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,1,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,1023,0.02976000060637792
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,127,0.013290667285521826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,1023,0.0174346665541331
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,127,0.012944000462690989
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,2047,0.05043200155099233
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,15,0.01292266696691513
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,1,0.013125333935022354
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,7,0.013082666943470636
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,15,0.015008000036080679
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,1,0.013002666334311167
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,15,0.013264000415802002
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,31,0.012997332960367203
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,3,0.014378666877746582
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,63,0.013045333325862885
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,7,0.013823999712864557
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,127,0.015061333775520325
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,127,0.013013333082199097
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,31,0.015066667149464289
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,255,0.0144213338692983
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,511,0.03070399910211563
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,511,0.01721599946419398
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,1023,0.04850133260091146
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,1023,0.031445334355036415
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,3,0.012970666090647379
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,2047,0.08634666601816814
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,2047,0.05035200218359629
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,1,0.01720000058412552
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,255,0.012944000462690989
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,3,0.019237333287795384
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,1,0.015141333142916361
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,3,0.01730666682124138
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,7,0.01918399954835574
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,7,0.017407999684413273
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,15,0.019253333409627277
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,15,0.017157333592573803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,31,0.019280000279347103
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,31,0.017152000218629837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,63,0.019130667050679524
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,63,0.017423999806245167
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,127,0.019237333287795384
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,255,0.01708799973130226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,255,0.03035733352104823
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,2047,0.0325546662012736
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,511,0.04860800007979075
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,511,0.03177600105603536
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,1023,0.051488002141316734
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,2047,0.08683733145395915
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,1,0.01722666621208191
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,1,0.019215999792019527
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,1,0.015109332899252573
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,63,0.014282666146755219
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,3,0.017375999440749485
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,7,0.01706133286158244
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,7,0.015034666905800501
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,1023,0.08468266328175862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,15,0.01505600040157636
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,2047,0.15639999508857727
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,31,0.017509333789348602
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,127,0.017263999829689663
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,31,0.015109332899252573
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,63,0.015178666760524115
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,63,0.017658667018016178
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,3,0.015205333630243937
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,255,0.015072000523408255
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,127,0.015568000574906668
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,511,0.03142400085926056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,1,0.01923199991385142
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,1,0.017829333742459614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,511,0.019130667050679524
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,3,0.01754133279124896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,127,0.01708799973130226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,7,0.01942933350801468
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,15,0.01708799973130226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,15,0.019679999599854153
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,31,0.019199999670187633
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,31,0.01718933383623759
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,3,0.019066666563351948
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,127,0.019130667050679524
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,127,0.017407999684413273
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,63,0.019904000063737232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,15,0.017173333714405697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,255,0.030597334106763203
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,255,0.017797333498795826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,511,0.049141332507133484
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,511,0.031514666974544525
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,1,0.02975466599067052
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,1,0.026341333985328674
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,3,0.030421334008375805
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,255,0.01716800034046173
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,7,0.025386666258176167
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,3,0.025797332326571148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,15,0.02972800036271413
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,15,0.025424001117547352
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,31,0.029472000896930695
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,31,0.026234666506449383
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,63,0.029391999046007793
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,63,0.02604266752799352
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,7,0.017338667064905167
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,127,0.03369066615899404
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,7,0.030437332888444264
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,127,0.027162666122118633
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,255,0.04974400003751119
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,255,0.03289599965016047
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,511,0.05097599824269613
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,1,0.02536533276240031
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,63,0.017445333302021027
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,1,0.021322667598724365
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,3,0.024495999018351238
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,7,0.024842667082945507
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,3,0.020597333709398907
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,31,0.025477332373460133
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,511,0.08507733543713887
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,15,0.025663999219735462
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,31,0.021205333371957142
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,15,0.021205333371957142
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,63,0.019744000087181728
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,127,0.02537599951028824
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,127,0.019845332950353622
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,255,0.021231998999913532
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,1,0.027456000447273254
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,63,0.025263999899228413
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,3,0.031386665999889374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,1,0.03182933231194814
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,3,0.027674667537212372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,7,0.027317332724730175
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,7,0.019952000429232914
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,7,0.03146666785081228
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,15,0.02739733209212621
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,31,0.03149333347876867
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,255,0.03155199935038885
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,31,0.02741333345572154
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,63,0.03152533372243246
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,127,0.03545066714286804
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,15,0.03151999910672506
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,1,0.05388266841570536
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,255,0.05190933247407278
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,63,0.02741333345572154
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,3,0.05380799869696299
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,3,0.04515733321507772
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,127,0.027509334186712902
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,7,0.053946668903032936
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,1,0.045797333121299744
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,15,0.05412266651789347
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,255,0.03545066714286804
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,31,0.054042667150497437
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,31,0.045797333121299744
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,63,0.05398933092753092
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,63,0.04413333535194397
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,7,0.045935998360315956
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,127,0.05611200133959452
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,1,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,3,0.009029333169261614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,7,0.00927466650803884
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,255,0.0888266662756602
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,3,0.009045333291093508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,15,0.00922133338948091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,255,0.05613866448402405
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,31,0.009162666896979014
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,63,0.009178666397929192
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,15,0.009477333476146063
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,127,0.04772266745567322
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,127,0.010549332946538925
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,63,0.009088000282645226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,255,0.009946666657924652
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,15,0.044106667240460716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,255,0.009029333169261614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,1023,0.011215999722480774
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,7,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,2047,0.01089599976936976
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,31,0.009045333291093508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,4095,0.01332266628742218
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,8191,0.017184000462293625
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,127,0.00892800030608972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,8191,0.015813333292802174
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,16383,0.033386667569478355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,511,0.011050666371981302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,1,0.008938666433095932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,32767,0.03425599883000056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,1023,0.011221333096424738
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,1,0.00916800027092298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,1,0.009242666885256767
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,3,0.009162666896979014
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,3,0.009941333283980688
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,7,0.009141333401203156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,7,0.00914666677514712
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,15,0.009253333633144697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,31,0.009072000160813332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,31,0.008938666433095932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,63,0.00891733355820179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,63,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,127,0.009029333169261614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,32767,0.051925331354141235
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,127,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,255,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,2047,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,511,0.010181333248813948
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,4095,0.013354666531085968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,15,0.009061333412925402
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,1023,0.010928000013033548
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,2047,0.01313599944114685
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,2047,0.011002667248249054
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,16383,0.019071999937295914
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,511,0.011039999624093374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,8191,0.015354666858911514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,255,0.009296000003814697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,16383,0.047925333182017006
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,511,0.011071999867757162
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,16383,0.03162133445342382
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,32767,0.08467732866605122
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,1023,0.011098666737476984
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,1,0.009258666386206945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,4095,0.013290667285521826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,3,0.009141333401203156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,4095,0.012975999464591345
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,7,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,8191,0.02962133288383484
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,15,0.00898133342464765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,31,0.009578666960199675
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,31,0.009877333417534828
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,63,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,63,0.009296000003814697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,127,0.00949866697192192
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,32767,0.049866666396458946
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,1,0.009317333499590555
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,255,0.009455999980370203
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,3,0.009706666693091393
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,1023,0.011567999919255575
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,1023,0.011098666737476984
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,2047,0.013690666606028875
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,7,0.009194666519761086
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,15,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,4095,0.015034666905800501
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,4095,0.029626667499542236
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,8191,0.030933332939942677
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,127,0.009877333417534828
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,255,0.009290666629870733
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,16383,0.0491839994986852
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,511,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,32767,0.08561600248018901
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,2047,0.01309866706530253
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,8191,0.047872001926104225
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,16383,0.08360000451405843
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,511,0.010853332777818045
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,1,0.042117332418759666
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,1,0.03322133421897888
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,3,0.04159999887148539
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,3,0.0315786674618721
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,32767,0.15588800112406412
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,7,0.03169599920511246
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,15,0.04172799984614054
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,15,0.03242133309443792
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,31,0.04177600145339966
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,31,0.03146666785081228
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,7,0.041663999358812966
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,63,0.04180799921353658
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,63,0.031717332700888314
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,127,0.043882668018341064
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,127,0.033200000723203026
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,1,0.05868266522884369
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,1,0.048623998959859215
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,3,0.05801600217819214
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,3,0.04814933240413666
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,7,0.058101331194241844
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,7,0.04983466863632202
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,15,0.058005332946777344
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,31,0.05876266459623972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,31,0.04984533290068308
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,63,0.05801066756248474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,63,0.048858667413393654
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,15,0.049866666396458946
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,127,0.06106133262316386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,127,0.05194133520126343
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,1,0.09897067149480183
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,1,0.08480532964070638
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,3,0.09763200084368388
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,7,0.09870400031407674
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,7,0.08467732866605122
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,15,0.09854400157928467
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,3,0.08483200271924336
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,31,0.09937600294748943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,63,0.09903466701507568
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,63,0.08462933699289958
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,31,0.08470400174458821
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,127,0.10242133339246114
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,127,0.08467732866605122
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,15,0.08462400237719218
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,float16,1,0.0765173335870107
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,1,0.05453333258628845
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,float16,3,0.07699733475844066
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,3,0.05568000177542368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,float16,7,0.07686399916807811
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,7,0.05603733162085215
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,float16,15,0.07640533149242401
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,15,0.05595199763774872
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,float16,31,0.07645866771539052
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,31,0.054234668612480164
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,float16,63,0.07657066484292348
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,63,0.05640000104904175
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,float16,1,0.10357333223025005
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,1,0.09297066926956177
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,float16,3,0.1036906639734904
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,float16,7,0.10550399621327718
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,3,0.09384533762931824
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,7,0.09360000491142273
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,float16,15,0.10513066252072652
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,15,0.09319466352462769
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,float16,31,0.1048959990342458
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,31,0.09286933143933614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,float16,63,0.10437333583831787
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,63,0.0929813285668691
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,float16,1,0.1830400029818217
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,1,0.15664533774058023
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,float16,3,0.18398400147755942
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,float16,7,0.18263999621073404
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,7,0.1567146678765615
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,1,0.009253333633144697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,1,0.01071999967098236
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,3,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,3,0.011066666493813196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,float16,15,0.18297600746154785
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,7,0.00916800027092298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,15,0.00898133342464765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,15,0.009242666885256767
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,31,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,31,0.010725333044926325
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,63,0.009269333134094873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,15,0.1566933294137319
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,3,0.15717333555221558
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,31,0.15659200151761374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,float16,31,0.18296533823013306
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,127,0.010288000106811523
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,7,0.00915733352303505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,511,0.01089599976936976
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,511,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,1023,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,1023,0.011157333850860596
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,2047,0.012991999586423239
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,2047,0.013077333569526672
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,4095,0.015157333264748255
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,4095,0.013786666095256805
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,63,0.010981333752473196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,8191,0.017130666722853977
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,16383,0.04920533299446106
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,127,0.00933333362142245
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,float16,63,0.18466132879257202
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,255,0.009343999748428663
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,1,0.00916800027092298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,3,0.009589333087205887
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,7,0.00922133338948091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,7,0.009242666885256767
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,3,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,15,0.009189333145817121
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,15,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,8191,0.03036266565322876
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,31,0.009578666960199675
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,63,0.00927466650803884
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,16383,0.03281066566705704
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,127,0.009317333499590555
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,127,0.009226666763424873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,255,0.009285333255926767
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,255,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,63,0.15646933515866598
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,63,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,2047,0.012965332716703415
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,1,0.009653333574533463
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,2047,0.012975999464591345
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,4095,0.030005333324273426
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,255,0.009050666665037474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,4095,0.015061333775520325
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,8191,0.03151999910672506
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,16383,0.08452266454696655
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,511,0.01091733326514562
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,1,0.010794666906197866
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,31,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,1023,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,3,0.010640000303586325
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,3,0.01090666651725769
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,7,0.009338666374484697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,7,0.009088000282645226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,15,0.011007999380429586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,15,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,31,0.010853332777818045
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,8191,0.04794666667779287
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,63,0.009296000003814697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,63,0.010938666760921478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,127,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,127,0.00919999989370505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,16383,0.049786667029062905
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,255,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,255,0.009952000031868616
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,511,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,1023,0.01321600005030632
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,1023,0.012821332861979803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,2047,0.029887999097506206
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,2047,0.015125333021084467
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,4095,0.047775998711586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,31,0.009370666618148485
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,8191,0.08266666531562805
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,8191,0.049072002371152244
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,511,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,511,0.011002667248249054
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,16383,0.15583466490109762
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,1,0.009269333134094873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,3,0.009861333295702934
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,7,0.009690666571259499
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,7,0.01033599985142549
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,15,0.00985599992175897
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,1,0.009248000259200731
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,31,0.00892800030608972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,31,0.009119999905427298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,63,0.00890666681031386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,63,0.009973333527644476
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,127,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,127,0.009269333134094873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,255,0.008885333314538002
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,255,0.009237333511312803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,15,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,1,0.010992000500361124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,1023,0.011359999577204386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,3,0.009039999917149544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,2047,0.012917333592971167
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,1023,0.01157333329319954
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,4095,0.015119999647140503
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,4095,0.03081600119670232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,4095,0.014277332772811254
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,8191,0.01722666621208191
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,8191,0.03275199979543686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,16383,0.04977599779764811
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,16383,0.03219199925661087
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,511,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,3,0.009077333534757296
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,1023,0.011589333415031433
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,7,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,7,0.009269333134094873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,15,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,15,0.00926399976015091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,31,0.008938666433095932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,31,0.00943999985853831
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,511,0.011034666250149408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,63,0.00898133342464765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,63,0.009098666409651438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,127,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,16383,0.08582400282224019
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,127,0.009045333291093508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,255,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,511,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,1023,0.012357333054145178
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,2047,0.012367999802033106
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,2047,0.013301332791646322
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,2047,0.013088000317414602
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,4095,0.02991466720898946
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,1,0.00922133338948091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,1,0.00898133342464765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,3,0.00903466654320558
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,8191,0.0314026673634847
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,511,0.011226666470368704
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,16383,0.04994133114814758
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,1023,0.011813333878914515
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,3,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,7,0.011120000233252844
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,4095,0.01515199989080429
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,7,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,8191,0.04850666721661886
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,255,0.009279999881982803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,15,0.009232000137368837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,31,0.010751999914646149
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,31,0.01089599976936976
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,63,0.009061333412925402
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,1,0.009621333330869675
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,127,0.010026666646202406
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,127,0.009258666386206945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,255,0.009573333586255709
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,255,0.009098666409651438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,511,0.010922666639089584
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,511,0.011120000233252844
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,1023,0.013248000293970108
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,1023,0.011711999773979187
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,2047,0.015034666905800501
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,4095,0.04804266492525736
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,16383,0.08567466338475545
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,4095,0.030458666384220123
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,63,0.009248000259200731
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,1,0.0107893335322539
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,3,0.010735999792814255
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,1,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,1,0.011066666493813196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,3,0.011141333729028702
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,15,0.010757333288590113
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,3,0.010922666639089584
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,7,0.011114666859308878
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,7,0.011141333729028702
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,15,0.011066666493813196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,15,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,8191,0.08330133557319641
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,31,0.011034666250149408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,16383,0.15687466661135355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,16383,0.08707200487454732
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,63,0.010981333752473196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,127,0.011338666081428528
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,127,0.011120000233252844
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,255,0.011152000476916632
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,255,0.010933333386977514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,511,0.013957332819700241
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,1023,0.02943466603755951
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,1023,0.01532799998919169
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,8191,0.04974400003751119
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,2047,0.04775466521581014
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,31,0.011034666250149408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,2047,0.031557333966096245
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,63,0.012138667205969492
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,4095,0.08390399813652039
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,8191,0.156741331020991
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,8191,0.08708266417185466
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,1,0.008805333326260248
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,1,0.009077333534757296
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,16383,0.30398400624593097
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,3,0.00892800030608972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,511,0.013151999562978745
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,7,0.009039999917149544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,15,0.008874666566650072
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,31,0.008922666932145754
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,31,0.009183999771873156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,63,0.008896000062425932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,127,0.008885333314538002
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,4095,0.049498667319615684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,127,0.009408000235756239
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,255,0.009103999783595404
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,2047,0.02937600016593933
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,16383,0.16190399726231894
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,3,0.00891733355820179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,1023,0.011488000551859537
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,7,0.009098666409651438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,2047,0.011237333218256632
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,15,0.00955200009047985
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,2047,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,63,0.009349333122372627
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,4095,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,8191,0.011642667154471079
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,16383,0.01533866673707962
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,255,0.009258666386206945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,511,0.010826667149861654
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,65535,0.02736533433198929
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,1023,0.011007999380429586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,1,0.008863999818762144
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,4095,0.011343999455372492
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,1,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,8191,0.01108266661564509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,3,0.00892800030608972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,16383,0.015146666516860327
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,7,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,15,0.009253333633144697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,7,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,32767,0.019248000035683315
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,31,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,63,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,63,0.00891733355820179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,127,0.00892800030608972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,127,0.008912000184257826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,255,0.00898133342464765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,511,0.01109333336353302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,3,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,1023,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,15,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,31,0.00916800027092298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,32767,0.019194666296243668
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,2047,0.010928000013033548
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,511,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,4095,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,65535,0.02741866558790207
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,8191,0.010928000013033548
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,16383,0.015290666371583939
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,32767,0.019472000499566395
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,511,0.01097600037852923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,65535,0.02327466756105423
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,1023,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,1,0.009093333035707474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,1,0.009141333401203156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,8191,0.010981333752473196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,255,0.009136000027259191
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,3,0.00891733355820179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,7,0.009205333267649015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,32767,0.019413333386182785
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,15,0.009050666665037474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,15,0.00921066664159298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,65535,0.022895999252796173
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,31,0.009226666763424873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,63,0.009232000137368837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,63,0.009232000137368837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,4095,0.011055999745925268
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,127,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,3,0.009141333401203156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,16383,0.01504533365368843
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,255,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,511,0.010965333630641302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,1023,0.010965333630641302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,31,0.008869333192706108
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,1023,0.010890666395425797
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,2047,0.01089599976936976
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,2047,0.010928000013033548
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,4095,0.011258666714032492
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,4095,0.010954666882753372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,8191,0.011637333780527115
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,8191,0.010933333386977514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,16383,0.016469333320856094
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,16383,0.015157333264748255
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,32767,0.019440000255902607
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,32767,0.017808000246683758
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,65535,0.03586666782697042
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,65535,0.02022933339079221
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,1,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,1,0.009637333452701569
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,2047,0.010741333166758219
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,3,0.00921066664159298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,7,0.009322666873534521
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,255,0.00898133342464765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,15,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,31,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,511,0.011007999380429586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,63,0.008912000184257826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,127,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,127,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,3,0.009103999783595404
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,7,0.009008000294367472
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,127,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,511,0.010154666379094124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,511,0.010901333143313726
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,7,0.008853333070874214
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,1023,0.010890666395425797
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,63,0.009269333134094873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,2047,0.010879999647537867
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,2047,0.010879999647537867
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,4095,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,8191,0.01328533391157786
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,255,0.009066666786869368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,16383,0.015194666882356008
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,15,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,1023,0.01121066634853681
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,32767,0.019199999670187633
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,65535,0.052111998200416565
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,4095,0.010949333508809408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,1,0.009050666665037474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,8191,0.013445333888133367
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,3,0.009285333255926767
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,255,0.009178666397929192
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,7,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,15,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,31,0.009045333291093508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,15,0.00960533320903778
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,31,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,63,0.008938666433095932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,65535,0.03555200000603994
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,63,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,1,0.009237333511312803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,255,0.008949333180983862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,255,0.008869333192706108
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,511,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,32767,0.033471999069054924
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,511,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,31,0.009077333534757296
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,1023,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,2047,0.01099733387430509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,3,0.009109333157539368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,4095,0.01116266722480456
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,8191,0.01312000056107839
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,8191,0.012240000069141388
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,16383,0.016095999628305435
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,16383,0.017242666333913803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,32767,0.02092266579469045
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,2047,0.011253333340088526
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,32767,0.02146666745344798
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,65535,0.023344000180562336
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,1,0.009029333169261614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,1,0.00903466654320558
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,3,0.009039999917149544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,3,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,7,0.009072000160813332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,7,0.009061333412925402
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,1023,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,65535,0.02351466566324234
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,127,0.008912000184257826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,31,0.00891733355820179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,63,0.008826666822036108
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,63,0.009226666763424873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,127,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,127,0.00921066664159298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,255,0.008933333059151968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,255,0.009226666763424873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,127,0.00921066664159298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,511,0.011114666859308878
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,1023,0.011087999989589056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,15,0.008938666433095932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,31,0.009098666409651438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,16383,0.017024000485738117
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,7,0.009226666763424873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,4095,0.01089599976936976
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,4095,0.01108266661564509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,8191,0.011424000064531961
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,8191,0.01108266661564509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,15,0.008879999940594038
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,4095,0.011007999380429586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,2047,0.01109333336353302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,65535,0.0355679988861084
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,65535,0.02057066683967908
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,1,0.008890666688481966
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,1,0.00891733355820179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,3,0.008874666566650072
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,16383,0.015461333096027374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,511,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,7,0.00903466654320558
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,32767,0.018506667266289394
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,15,0.008922666932145754
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,2047,0.010837333897749582
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,15,0.009413333609700203
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,31,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,31,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,63,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,63,0.008901333436369896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,127,0.00884799969693025
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,127,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,255,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,255,0.00914666677514712
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,511,0.009946666657924652
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,1023,0.011007999380429586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,1023,0.010911999891201654
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,1023,0.01108266661564509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,2047,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,4095,0.010933333386977514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,4095,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,8191,0.013167999684810638
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,8191,0.013290667285521826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,16383,0.01720533271630605
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,16383,0.015813333292802174
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,32767,0.0335359995563825
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,32767,0.019333332777023315
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,65535,0.051327998439470925
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,511,0.010821333775917688
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,65535,0.03570666660865148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,3,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,1,0.00898133342464765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,32767,0.01923199991385142
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,7,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,7,0.009103999783595404
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,7,0.009237333511312803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,15,0.00890666681031386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,15,0.009306666751702627
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,31,0.00921066664159298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,63,0.008949333180983862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,63,0.008885333314538002
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,127,0.008896000062425932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,255,0.00903466654320558
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,1,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,16383,0.01540800059835116
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,511,0.01090666651725769
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,511,0.011039999624093374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,1023,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,31,0.008933333059151968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,2047,0.011242666592200598
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,127,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,4095,0.013093333691358566
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,4095,0.013050666699806849
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,8191,0.01534933348496755
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,255,0.00903466654320558
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,16383,0.0317493329445521
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,16383,0.017093333105246227
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,32767,0.05003199974695841
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,32767,0.03365866591533025
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,65535,0.08617066343625386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,65535,0.052202666799227394
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,1,0.010751999914646149
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,1,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,3,0.010437333335479101
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,3,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,8191,0.015173333386580149
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,2047,0.010981333752473196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,7,0.011114666859308878
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,15,0.009312000125646591
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,15,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,1023,0.011087999989589056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,31,0.010768000036478043
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,31,0.010751999914646149
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,63,0.010949333508809408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,63,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,127,0.010981333752473196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,127,0.009594666461149851
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,255,0.010933333386977514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,255,0.011066666493813196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,511,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,511,0.01109333336353302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,1023,0.013034666577974955
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,2047,0.015263999501864115
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,2047,0.01314666618903478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,4095,0.030405332644780476
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,4095,0.017194667210181553
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,2047,0.011253333340088526
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,8191,0.049285332361857094
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,8191,0.031856000423431396
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,1,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,1,0.009359999870260557
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,3,0.009317333499590555
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,1023,0.012896000097195307
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,7,0.009029333169261614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,7,0.010832000523805618
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,15,0.00949866697192192
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,15,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,31,0.011002667248249054
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,31,0.008879999940594038
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,63,0.009893333539366722
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,63,0.010149333626031876
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,127,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,255,0.009589333087205887
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,3,0.010832000523805618
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,255,0.00922133338948091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,511,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,3,0.008912000184257826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,7,0.009253333633144697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,1023,0.014293332894643148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,2047,0.029520000020662945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,2047,0.015200000256299973
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,4095,0.03145066648721695
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,127,0.009573333586255709
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,1,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,8191,0.049925332268079124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,1,0.010965333630641302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,3,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,511,0.011157333850860596
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,7,0.011039999624093374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,15,0.011178666104873022
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,4095,0.04795200129350027
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,31,0.011264000087976456
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,31,0.01109333336353302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,63,0.011477333803971609
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,63,0.011141333729028702
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,127,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,3,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,255,0.011050666371981302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,511,0.01440000037352244
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,8191,0.08467732866605122
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,511,0.013002666334311167
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,3,0.010911999891201654
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,1023,0.02945599953333537
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,1023,0.01509333277742068
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,2047,0.04806933303674062
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,2047,0.03146666785081228
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,1023,0.013034666577974955
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,4095,0.08457600076993306
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,8191,0.15775466958681741
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,8191,0.08830400307973225
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,1,0.013210666676362356
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,3,0.015125333021084467
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,7,0.01504533365368843
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,7,0.012858666479587555
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,7,0.011055999745925268
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,15,0.014090667168299357
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,15,0.013002666334311167
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,31,0.014378666877746582
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,31,0.01302933320403099
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,63,0.014378666877746582
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,63,0.01314666618903478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,127,0.015216000378131866
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,127,0.01309866706530253
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,3,0.012991999586423239
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,255,0.014922666052977243
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,511,0.02945599953333537
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,511,0.01709866647919019
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,1023,0.04816000163555145
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,1023,0.03146666785081228
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,2047,0.084714670976003
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,1,0.013066666821638743
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,2047,0.0499946673711141
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,255,0.01309866706530253
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,4095,0.15874666968981424
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,4095,0.08678932984670003
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,1,0.009029333169261614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,1,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,3,0.008949333180983862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,3,0.00927466650803884
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,7,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,7,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,15,0.009232000137368837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,15,0.009258666386206945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,127,0.011114666859308878
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,31,0.008938666433095932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,63,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,63,0.009269333134094873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,127,0.00916800027092298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,8191,0.3051786621411641
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,255,0.009109333157539368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,511,0.011098666737476984
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,511,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,31,0.009546666716535887
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,15,0.011226666470368704
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,1023,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,1023,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,127,0.009066666786869368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,2047,0.01097600037852923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,4095,0.01109333336353302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,4095,0.010954666882753372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,8191,0.012389333297808966
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,8191,0.01293333371480306
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,16383,0.01721599946419398
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,8191,0.15946666399637857
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,32767,0.020362666497627895
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,32767,0.01912533367673556
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,65535,0.035760000348091125
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,65535,0.021327999730904896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,255,0.011285333583752314
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,1,0.009098666409651438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,1,0.00892800030608972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,3,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,3,0.009093333035707474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,7,0.009109333157539368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,255,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,31,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,31,0.008933333059151968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,16383,0.01706133286158244
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,63,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,127,0.00915733352303505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,127,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,255,0.008885333314538002
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,255,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,7,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,511,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,1023,0.01109333336353302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,15,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,2047,0.010911999891201654
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,2047,0.011087999989589056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,4095,0.010981333752473196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,63,0.008890666688481966
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,4095,0.011039999624093374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,8191,0.013290667285521826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,16383,0.01722666621208191
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,4095,0.04943466683228811
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,32767,0.03209600100914637
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,32767,0.01934933289885521
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,2047,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,15,0.010853332777818045
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,1,0.009232000137368837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,1023,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,3,0.008912000184257826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,3,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,7,0.009114666531483332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,7,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,15,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,15,0.00898133342464765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,31,0.009237333511312803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,31,0.00898133342464765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,63,0.009717333440979322
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,65535,0.050944000482559204
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,127,0.009258666386206945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,65535,0.03477866699298223
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,127,0.009098666409651438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,255,0.009173333023985228
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,511,0.009808000177145004
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,1023,0.011114666859308878
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,8191,0.01310933381319046
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,1023,0.01239466667175293
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,16383,0.015781333049138386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,2047,0.011077333241701126
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,4095,0.01302933320403099
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,511,0.011034666250149408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,63,0.009045333291093508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,8191,0.015125333021084467
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,16383,0.01716800034046173
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,1,0.009248000259200731
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,511,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,65535,0.0867146650950114
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,2047,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,65535,0.05218133330345154
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,1,0.009103999783595404
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,1,0.009632000078757605
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,4095,0.01312000056107839
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,3,0.009066666786869368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,16383,0.031397332747777305
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,255,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,7,0.009103999783595404
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,7,0.008890666688481966
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,32767,0.04993600149949392
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,31,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,63,0.009082666908701261
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,63,0.009183999771873156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,127,0.009253333633144697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,127,0.00933333362142245
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,255,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,3,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,511,0.011039999624093374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,511,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,1023,0.011039999624093374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,32767,0.03377600014209747
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,1023,0.011349332829316458
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,15,0.008842666943868002
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,2047,0.011296000331640244
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,4095,0.014127999544143677
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,4095,0.012928000340859095
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,8191,0.029333333174387615
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,8191,0.015040000279744467
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,255,0.009093333035707474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,16383,0.047877331574757896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,8191,0.015029333531856537
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,32767,0.08411733309427898
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,32767,0.048885335524876915
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,65535,0.15599999825159708
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,65535,0.08644266923268636
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,2047,0.012965332716703415
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,1,0.011141333729028702
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,3,0.011007999380429586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,3,0.010992000500361124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,7,0.010933333386977514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,15,0.010837333897749582
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,16383,0.03105599929889043
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,31,0.012085333466529846
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,31,0.011050666371981302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,63,0.010954666882753372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,63,0.011050666371981302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,127,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,255,0.011152000476916632
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,7,0.010928000013033548
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,255,0.011066666493813196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,15,0.010890666395425797
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,511,0.011173332730929056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,1023,0.014442666123310724
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,1023,0.012970666090647379
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,15,0.00922133338948091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,2047,0.02977599948644638
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,2047,0.017242666333913803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,127,0.01119999960064888
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,4095,0.04789333542188009
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,4095,0.0313226655125618
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,1,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,511,0.013104000439246496
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,3,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,7,0.011109333485364914
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,7,0.010992000500361124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,31,0.009381333366036415
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,15,0.010949333508809408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,31,0.011178666104873022
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,31,0.010863999525705973
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,63,0.01116266722480456
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,127,0.011349332829316458
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,127,0.011215999722480774
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,3,0.011648000528415045
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,255,0.011274666835864386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,255,0.011120000233252844
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,511,0.013317332913478216
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,511,0.013093333691358566
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,15,0.011114666859308878
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,1,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,1023,0.015392000476519266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,2047,0.048800001541773476
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,2047,0.03179733455181122
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,1,0.011424000064531961
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,1,0.013962666193644205
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,1,0.013056000073750814
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,3,0.01488000030318896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,7,0.013786666095256805
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,7,0.013045333325862885
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,15,0.0143306665122509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,15,0.01313599944114685
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,4095,0.05157866577307383
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,31,0.013088000317414602
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,63,0.015216000378131866
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,63,0.013023999830087027
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,127,0.015077333897352219
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,127,0.013210666676362356
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,255,0.015146666516860327
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,63,0.011002667248249054
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,511,0.01708799973130226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,3,0.01321600005030632
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,1023,0.04916266600290934
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,1023,0.03046400099992752
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,255,0.012981332838535309
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,2047,0.08587732911109924
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,2047,0.05117866893609365
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,4095,0.08610133330027263
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,1,0.01926933353145917
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,4095,0.08716799815495808
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,1,0.017184000462293625
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,3,0.017194667210181553
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,3,0.01926400015751521
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,7,0.017055999487638474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,15,0.019082666685183842
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,511,0.03052799900372823
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,31,0.0174346665541331
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,1023,0.03182933231194814
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,63,0.019199999670187633
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,127,0.017210666090250015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,31,0.014058666924635569
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,255,0.029477333029111225
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,255,0.017093333105246227
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,7,0.019130667050679524
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,511,0.04794133206208547
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,127,0.019194666296243668
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,1023,0.08504000306129456
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,31,0.01926400015751521
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,1023,0.0516480008761088
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,2047,0.15677332878112793
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,2047,0.08810133735338847
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,63,0.017071999609470367
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,4095,0.299727996190389
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,4095,0.1625333329041799
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,15,0.017045332739750545
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,1,0.013104000439246496
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,4095,0.15846400459607443
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,3,0.013178666432698568
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,3,0.013072000195582708
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,7,0.013050666699806849
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,7,0.012928000340859095
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,15,0.013104000439246496
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,31,0.013151999562978745
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,511,0.032746667663256325
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,63,0.013167999684810638
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,127,0.013002666334311167
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,255,0.013002666334311167
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,255,0.012965332716703415
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,15,0.014256000518798828
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,511,0.015077333897352219
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,31,0.013023999830087027
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,63,0.013082666943470636
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,511,0.014959999670584997
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,127,0.013141332815090815
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,1023,0.01708799973130226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,1,0.013408000270525614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,3,0.014938666174809137
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,3,0.01313599944114685
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,7,0.01440000037352244
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,7,0.013045333325862885
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,15,0.015194666882356008
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,1,0.013066666821638743
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,31,0.01505600040157636
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,31,0.013125333935022354
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,63,0.014314666390419006
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,63,0.013381333400805792
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,1,0.015061333775520325
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,127,0.014981333166360855
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,127,0.013370666652917862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,255,0.013066666821638743
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,511,0.030565333863099415
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,15,0.01301866645614306
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,1023,0.03140799949566523
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,1,0.019248000035683315
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,1,0.017530667285124462
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,3,0.019173332800467808
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,255,0.015098666151364645
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,3,0.01722666621208191
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,7,0.019280000279347103
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,7,0.01714666684468587
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,15,0.01912533367673556
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,1023,0.04975999891757965
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,15,0.017221332838137943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,31,0.019296000401178997
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,31,0.017429333180189133
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,63,0.019253333409627277
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,63,0.017055999487638474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,127,0.017210666090250015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,255,0.030576000610987347
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,127,0.019280000279347103
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,511,0.01708799973130226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,511,0.048895999789237976
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,1023,0.031983998914559685
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,511,0.03136533250411352
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,1,0.02934933453798294
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,1,0.026416001220544178
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,1023,0.08477866649627686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,1023,0.0503359983364741
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,255,0.017530667285124462
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,3,0.02938133229811986
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,15,0.029877332349618275
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,3,0.025557334224383037
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,7,0.029482667644818623
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,31,0.027466667195161183
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,63,0.02569066733121872
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,63,0.029477333029111225
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,31,0.02945599953333537
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,127,0.025519999365011852
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,127,0.03250666707754135
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,7,0.025487999121348064
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,255,0.03249600032965342
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,255,0.04950400193532308
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,1023,0.0867199997107188
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,1,0.017114666601022083
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,511,0.08538132905960083
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,1,0.015184000134468079
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,3,0.015125333021084467
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,7,0.017114666601022083
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,1023,0.15665599703788757
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,511,0.050661335388819374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,15,0.025648000339667004
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,15,0.01691199963291486
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,31,0.017157333592573803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,3,0.01725333308180173
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,63,0.015168000012636185
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,7,0.015082667271296183
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,127,0.017242666333913803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,127,0.015040000279744467
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,15,0.015072000523408255
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,255,0.01738133281469345
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,31,0.015072000523408255
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,255,0.015119999647140503
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,63,0.01709866647919019
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,511,0.031386665999889374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,511,0.019141333798567455
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,1,0.01718933383623759
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,3,0.01916266605257988
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,3,0.017898666361967724
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,7,0.019445333629846573
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,7,0.01717866708834966
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,15,0.01714133347074191
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,1,0.019205333044131596
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,31,0.01941866676012675
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,31,0.017637333522240322
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,63,0.019343999524911244
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,63,0.017237332959969837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,127,0.019253333409627277
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,127,0.017749333133300144
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,255,0.031189332405726116
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,255,0.0173333336909612
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,511,0.049829334020614624
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,511,0.03217600037654241
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,1,0.02752000093460083
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,3,0.03146666785081228
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,3,0.027466667195161183
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,7,0.03143466760714849
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,15,0.019167999426523846
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,7,0.02773333340883255
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,15,0.0315733328461647
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,1,0.03169066707293192
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,31,0.03152533372243246
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,31,0.02739199995994568
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,63,0.03145066648721695
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,63,0.027637332677841187
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,127,0.03558400024970373
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,15,0.02739733209212621
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,255,0.0359946663180987
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,1,0.05394133428732554
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,511,0.053898667295773826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,511,0.08845333258310954
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,1,0.045594667394955955
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,3,0.054330666859944664
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,127,0.028101332485675812
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,7,0.0539626677831014
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,3,0.045791998505592346
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,255,0.05194666484991709
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,15,0.04446933170159658
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,31,0.054287999868392944
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,7,0.04576533536116282
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,63,0.05435200035572052
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,63,0.04562133550643921
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,15,0.05395199855168661
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,1,0.009338666374484697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,1,0.009008000294367472
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,3,0.00926399976015091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,3,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,7,0.00921066664159298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,31,0.046037331223487854
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,127,0.05643199880917867
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,7,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,15,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,31,0.00903466654320558
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,255,0.05493866900602976
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,63,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,127,0.009077333534757296
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,127,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,511,0.09109866619110107
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,255,0.009029333169261614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,511,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,511,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,31,0.00966933307548364
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,1023,0.011109333485364914
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,15,0.010922666639089584
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,2047,0.011141333729028702
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,255,0.00922133338948091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,2047,0.011125333607196808
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,4095,0.011205332974592844
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,8191,0.015306666493415833
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,63,0.009423999736706415
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,16383,0.017781333376963932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,16383,0.01714666684468587
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,32767,0.03352533280849457
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,32767,0.021183999876181286
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,65535,0.052655999859174095
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,255,0.08842666943868001
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,1,0.008858666444818178
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,1023,0.01099733387430509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,3,0.009039999917149544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,4095,0.01109333336353302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,1,0.009088000282645226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,7,0.009050666665037474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,15,0.008933333059151968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,15,0.00927466650803884
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,511,0.16065067052841187
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,31,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,65535,0.03509333233038584
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,63,0.00938666673998038
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,127,0.009258666386206945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,3,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,63,0.00943999985853831
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,127,0.009397333487868309
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,255,0.009066666786869368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,511,0.010981333752473196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,127,0.0479360024134318
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,511,0.01090666651725769
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,2047,0.011258666714032492
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,1023,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,2047,0.011066666493813196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,4095,0.013125333935022354
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,8191,0.01523200049996376
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,8191,0.015061333775520325
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,16383,0.03164800008138021
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,16383,0.01736533393462499
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,31,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,32767,0.050197333097457886
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,1023,0.01119999960064888
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,65535,0.08669333656628926
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,65535,0.051488002141316734
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,1,0.00891733355820179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,3,0.00903466654320558
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,3,0.009029333169261614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,4095,0.012906666845083237
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,7,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,15,0.008949333180983862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,15,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,31,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,8191,0.015013333410024643
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,31,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,63,0.009253333633144697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,127,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,32767,0.03381866713364919
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,255,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,255,0.008949333180983862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,511,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,511,0.011002667248249054
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,1023,0.01097600037852923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,1023,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,7,0.009082666908701261
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,2047,0.011349332829316458
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,7,0.009066666786869368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,4095,0.013658666362365087
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,4095,0.013125333935022354
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,8191,0.015157333264748255
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,63,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,16383,0.030437332888444264
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,1,0.00927466650803884
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,32767,0.04855466882387797
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,65535,0.1567253371079763
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,65535,0.08643733461697896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,1,0.008879999940594038
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,1,0.009941333283980688
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,255,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,3,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,8191,0.02980799973011017
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,127,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,15,0.009082666908701261
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,15,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,31,0.00914666677514712
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,31,0.009194666519761086
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,63,0.009098666409651438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,2047,0.011034666250149408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,127,0.009072000160813332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,127,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,3,0.009061333412925402
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,16383,0.04786133269468943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,7,0.00891733355820179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,255,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,511,0.010960000256697336
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,32767,0.08469866712888081
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,1023,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,2047,0.013797332843144735
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,4095,0.02957333376010259
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,8191,0.04786666731039683
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,8191,0.03065066784620285
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,511,0.010949333508809408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,1023,0.011626667032639185
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,16383,0.08469333251317342
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,16383,0.04956800242265066
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,32767,0.08505599697430928
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,4095,0.015146666516860327
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,65535,0.3003786603609721
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,65535,0.15731733043988547
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,7,0.00960533320903778
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,2047,0.013072000195582708
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,32767,0.15593066811561584
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,63,0.009279999881982803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,255,0.009066666786869368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,1,0.021216000119845074
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,3,0.025386666258176167
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,7,0.02532266577084859
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,1,0.025493333737055462
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,15,0.02497066557407379
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,3,0.021168000996112823
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,15,0.021312000850836437
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,31,0.025093334416548412
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,31,0.021242665747801464
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,7,0.021141332884629566
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,63,0.025568000972270966
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,63,0.021194666624069214
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,127,0.025029333929220837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,127,0.02123733361562093
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,255,0.03141333411137263
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,255,0.021141332884629566
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,1,0.027877333263556164
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,3,0.03146666785081228
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,7,0.03142933299144109
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,7,0.027429332335789997
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,1,0.03146666785081228
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,15,0.03145066648721695
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,15,0.028431999186674755
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,3,0.02762666592995326
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,31,0.03146133323510488
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,31,0.02770666778087616
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,63,0.03153600047032038
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,63,0.027429332335789997
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,127,0.035536001125971474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,127,0.0278613343834877
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,255,0.05198400219281515
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,1,0.05809600154558817
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,1,0.04853333532810211
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,3,0.058042665322621666
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,3,0.04779199759165446
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,7,0.057962665955225624
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,7,0.04822933177153269
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,255,0.036602665980656944
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,15,0.04791999856630961
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,15,0.058005332946777344
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,31,0.058133333921432495
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,31,0.04773866633574168
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,63,0.05804799993832906
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,127,0.0529013325770696
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,63,0.04772266745567322
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,127,0.060826669136683144
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,1,0.09711466232935588
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,255,0.09311999877293904
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,255,0.061861331264177956
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,1,0.08474666873613994
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,3,0.09867200255393982
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,7,0.09752532839775085
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,3,0.08473599950472514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,7,0.08477333188056946
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,15,0.09745599826176961
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,31,0.08470933636029561
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,15,0.08482133348782857
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,63,0.099263995885849
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,31,0.09776000181833903
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,63,0.0846026639143626
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,127,0.10222400228182475
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,127,0.08545600374539693
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,255,0.16540799538294473
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,255,0.1004960040251414
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,1,0.04165866722663244
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,1,0.03213333338499069
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,3,0.04162666698296865
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,3,0.03178133318821589
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,7,0.04162133236726125
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,7,0.033520000676314034
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,15,0.04192533095677694
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,31,0.04181866844495138
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,15,0.031530665854612984
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,31,0.032485333581765495
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,63,0.031727999448776245
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,63,0.04215466479460398
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,127,0.043824002146720886
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,127,0.033546666304270424
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,1,0.055957332253456116
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,1,0.047882666190465294
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,3,0.05596266686916351
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,7,0.05592533449331919
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,7,0.04772266745567322
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,3,0.047781333327293396
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,15,0.05630399783452352
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,15,0.04789866507053375
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,31,0.056128000219662987
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,31,0.04764799773693085
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,63,0.05608533322811127
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,63,0.04786666731039683
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,1,0.10402133067448933
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,127,0.060346667965253196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,127,0.051813334226608276
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,1,0.0918933351834615
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,3,0.10518399874369304
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,3,0.0925333301226298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,7,0.09314133723576863
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,15,0.10482133428255717
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,31,0.10541866223017375
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,7,0.1051573355992635
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,31,0.09250666697820027
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,63,0.10424000024795532
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,63,0.09318400422732036
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,15,0.09294399619102478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,127,0.09437867005666097
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,3,0.18332799275716147
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,1,0.18434667587280273
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,127,0.11103467146555583
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,3,0.15651200215021768
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,1,0.15677332878112793
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,7,0.15640532970428467
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,7,0.1834239959716797
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,15,0.18293333053588867
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,15,0.15639467040697733
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,1,0.009279999881982803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,31,0.18481600284576416
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,3,0.009786666681369146
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,7,0.009824000298976898
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,7,0.009994666402538618
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,15,0.009733333562811216
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,15,0.009658666948477427
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,31,0.009072000160813332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,1,0.00926399976015091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,63,0.009130666653315226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,63,0.009621333330869675
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,127,0.009381333366036415
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,3,0.009663999701539675
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,63,0.18402665853500366
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,255,0.010015999898314476
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,255,0.00926399976015091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,511,0.010965333630641302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,511,0.011002667248249054
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,1023,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,1023,0.010885333021481832
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,2047,0.011055999745925268
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,2047,0.010981333752473196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,4095,0.013866666704416275
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,4095,0.01303999995191892
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,8191,0.01699200024207433
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,127,0.010048000141978264
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,16383,0.03324799984693527
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,127,0.19150400161743164
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,16383,0.019610666980346043
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,127,0.15845866998036703
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,31,0.009205333267649015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,3,0.008821333448092142
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,3,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,7,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,8191,0.016143999993801117
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,7,0.00926399976015091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,1,0.009082666908701261
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,31,0.15642133355140686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,1,0.009226666763424873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,32767,0.0352906659245491
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,31,0.009205333267649015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,63,0.008933333059151968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,31,0.009082666908701261
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,127,0.00916800027092298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,255,0.01027199998497963
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,255,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,511,0.011034666250149408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,511,0.010853332777818045
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,1023,0.011221333096424738
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,1023,0.010965333630641302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,2047,0.011333333949247995
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,32767,0.05147199829419454
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,4095,0.01444799949725469
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,4095,0.01314666618903478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,63,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,8191,0.03038399914900462
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,8191,0.015098666151364645
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,16383,0.04916266600290934
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,16383,0.03146666785081228
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,32767,0.08567999800046285
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,32767,0.049839998284975685
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,1,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,1,0.009130666653315226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,7,0.00921066664159298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,7,0.00919999989370505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,63,0.156549334526062
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,15,0.009194666519761086
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,127,0.008896000062425932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,15,0.008933333059151968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,31,0.009893333539366722
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,63,0.00890666681031386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,31,0.008912000184257826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,127,0.009994666402538618
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,3,0.009088000282645226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,255,0.00985599992175897
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,511,0.01089599976936976
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,511,0.011002667248249054
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,1023,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,1023,0.011264000087976456
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,63,0.00949866697192192
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,2047,0.013013333082199097
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,3,0.009279999881982803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,4095,0.030042665700117748
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,255,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,8191,0.047824000318845115
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,16383,0.08473066488901775
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,16383,0.050000001986821495
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,2047,0.01320533330241839
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,127,0.00921066664159298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,15,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,15,0.009103999783595404
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,1,0.009194666519761086
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,8191,0.03149333347876867
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,3,0.010762666662534079
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,7,0.009232000137368837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,7,0.009279999881982803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,15,0.009098666409651438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,32767,0.15666666626930237
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,15,0.009999999776482582
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,2047,0.010954666882753372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,32767,0.0865760048230489
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,1,0.010250666489203772
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,63,0.009258666386206945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,3,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,127,0.009296000003814697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,255,0.00915733352303505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,255,0.00926399976015091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,511,0.010949333508809408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,511,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,1023,0.01312000056107839
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,1023,0.011920000116030375
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,4095,0.015184000134468079
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,2047,0.015066667149464289
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,63,0.011071999867757162
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,4095,0.04752533137798309
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,4095,0.03010133405526479
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,127,0.011071999867757162
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,8191,0.048858667413393654
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,8191,0.08370666702588399
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,16383,0.15592533349990845
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,16383,0.08481599887212117
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,32767,0.2988640069961548
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,1,0.010847999403874079
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,3,0.009029333169261614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,3,0.009285333255926767
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,7,0.010341333225369453
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,31,0.009999999776482582
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,15,0.009530666594703993
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,31,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,31,0.009039999917149544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,63,0.009066666786869368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,63,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,1,0.00891733355820179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,127,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,32767,0.1567093332608541
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,255,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,511,0.010992000500361124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,7,0.009039999917149544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,31,0.010362666721145311
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,1023,0.011087999989589056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,15,0.009237333511312803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,2047,0.013280000537633896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,2047,0.012346666306257248
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,4095,0.015087999403476715
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,4095,0.013861333330472311
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,8191,0.02945599953333537
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,127,0.009103999783595404
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,16383,0.05003199974695841
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,16383,0.03180799881617228
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,1,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,1,0.009056000038981438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,3,0.00921066664159298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,7,0.009136000027259191
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,1023,0.011173332730929056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,15,0.009136000027259191
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,15,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,31,0.00898133342464765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,31,0.009183999771873156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,63,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,8191,0.017082666357358296
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,63,0.00916800027092298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,127,0.00921066664159298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,255,0.009306666751702627
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,255,0.009317333499590555
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,7,0.00903466654320558
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,1023,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,1023,0.011071999867757162
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,2047,0.013418667018413544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,127,0.00922133338948091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,2047,0.013114667187134424
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,3,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,4095,0.029951999584833782
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,511,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,8191,0.04880533119042715
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,8191,0.03141333411137263
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,16383,0.08531199892361958
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,1,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,1,0.010154666379094124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,3,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,3,0.010965333630641302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,255,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,7,0.010863999525705973
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,2047,0.029333333174387615
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,15,0.011050666371981302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,31,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,16383,0.049882665276527405
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,31,0.011007999380429586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,63,0.010629333555698395
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,63,0.01110400011142095
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,127,0.009242666885256767
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,255,0.011034666250149408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,7,0.01099733387430509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,255,0.009189333145817121
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,4095,0.01516266663869222
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,511,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,1023,0.013093333691358566
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,15,0.010922666639089584
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,1023,0.012997332960367203
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,2047,0.02942933390537898
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,2047,0.01498666654030482
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,4095,0.0315786674618721
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,127,0.010960000256697336
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,8191,0.08372267087300618
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,8191,0.05003733436266581
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,16383,0.08681066830952962
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,511,0.011007999380429586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,1,0.00919999989370505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,3,0.009365333244204521
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,3,0.009248000259200731
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,7,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,15,0.009242666885256767
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,4095,0.047882666190465294
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,15,0.00921066664159298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,31,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,63,0.00919999989370505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,1,0.009226666763424873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,127,0.009119999905427298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,127,0.009183999771873156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,511,0.01108266661564509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,255,0.00949866697192192
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,511,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,511,0.01108266661564509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,1023,0.011050666371981302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,2047,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,31,0.009189333145817121
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,2047,0.01108266661564509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,16383,0.1567200024922689
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,4095,0.011039999624093374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,8191,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,8191,0.011477333803971609
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,7,0.009322666873534521
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,16383,0.015109332899252573
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,32767,0.019226666539907455
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,1023,0.011205332974592844
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,65535,0.027482666075229645
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,1,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,1,0.009375999992092451
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,4095,0.011007999380429586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,3,0.009109333157539368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,3,0.009269333134094873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,255,0.008922666932145754
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,7,0.009141333401203156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,511,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,15,0.009285333255926767
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,31,0.00921066664159298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,32767,0.01937066639463107
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,63,0.009178666397929192
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,65535,0.02752000093460083
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,127,0.008874666566650072
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,127,0.009056000038981438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,255,0.008922666932145754
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,255,0.009050666665037474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,63,0.008922666932145754
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,511,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,7,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,2047,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,2047,0.010965333630641302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,4095,0.01097600037852923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,63,0.009205333267649015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,8191,0.011247999966144562
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,8191,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,511,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,1023,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,16383,0.015008000036080679
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,32767,0.01926933353145917
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,16383,0.01524266724785169
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,65535,0.02332266668478648
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,65535,0.02329600105683009
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,1,0.009029333169261614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,4095,0.011039999624093374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,3,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,3,0.009050666665037474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,7,0.009338666374484697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,16383,0.015285332997639975
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,15,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,15,0.008842666943868002
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,31,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,32767,0.01945066700379054
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,31,0.008933333059151968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,15,0.00903466654320558
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,63,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,63,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,31,0.010234666367371878
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,255,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,127,0.00892800030608972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,511,0.011087999989589056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,511,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,1023,0.010911999891201654
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,1023,0.011034666250149408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,2047,0.011039999624093374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,2047,0.01089599976936976
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,4095,0.011152000476916632
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,1023,0.010911999891201654
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,8191,0.01091733326514562
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,127,0.008858666444818178
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,255,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,32767,0.019167999426523846
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,7,0.00892800030608972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,32767,0.018197332819302876
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,65535,0.03583466758330663
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,65535,0.019589333484570186
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,8191,0.011242666592200598
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,1,0.009375999992092451
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,3,0.008863999818762144
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,3,0.009253333633144697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,7,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,7,0.009232000137368837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,15,0.009312000125646591
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,1,0.008933333059151968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,31,0.009445333232482275
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,63,0.009999999776482582
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,63,0.00966933307548364
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,127,0.008890666688481966
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,127,0.010112000008424124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,1,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,255,0.009296000003814697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,511,0.011136000355084738
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,511,0.01099733387430509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,1023,0.011125333607196808
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,2047,0.010928000013033548
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,16383,0.015210667004187902
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,2047,0.011039999624093374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,4095,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,4095,0.011285333583752314
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,8191,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,255,0.009248000259200731
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,16383,0.01552533358335495
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,16383,0.015285332997639975
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,32767,0.021210665504137676
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,32767,0.019466667125622433
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,1023,0.011194666226704916
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,65535,0.025242666403452556
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,1,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,31,0.009050666665037474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,3,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,7,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,7,0.009039999917149544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,15,0.009109333157539368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,15,0.009077333534757296
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,15,0.009050666665037474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,31,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,31,0.00903466654320558
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,63,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,127,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,127,0.008874666566650072
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,16383,0.015205333630243937
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,255,0.008890666688481966
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,255,0.009082666908701261
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,511,0.010965333630641302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,1023,0.011098666737476984
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,1,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,1023,0.011120000233252844
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,3,0.009066666786869368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,2047,0.011071999867757162
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,4095,0.011136000355084738
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,2047,0.011237333218256632
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,4095,0.012240000069141388
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,4095,0.010885333021481832
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,8191,0.011322667201360067
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,16383,0.015509333461523056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,32767,0.019578666736682255
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,32767,0.018405333161354065
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,511,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,65535,0.03719999889532725
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,1,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,1,0.00961599995692571
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,3,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,63,0.009061333412925402
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,3,0.009248000259200731
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,7,0.009119999905427298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,7,0.009183999771873156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,8191,0.01157333329319954
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,15,0.00898133342464765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,31,0.00890666681031386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,31,0.00891733355820179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,8191,0.012026666353146235
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,16383,0.015119999647140503
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,255,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,255,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,65535,0.023498666783173878
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,511,0.011999999483426413
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,1023,0.011087999989589056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,1023,0.011007999380429586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,2047,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,127,0.009178666397929192
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,65535,0.019850666324297588
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,4095,0.010879999647537867
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,8191,0.01369599997997284
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,511,0.011002667248249054
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,8191,0.013066666821638743
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,15,0.009205333267649015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,16383,0.015157333264748255
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,32767,0.03348266581694285
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,63,0.00922133338948091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,65535,0.051589335004488625
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,65535,0.03484266748030981
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,4095,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,1,0.009653333574533463
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,3,0.009573333586255709
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,16383,0.01704000060757001
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,3,0.01091733326514562
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,63,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,127,0.009349333122372627
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,7,0.011205332974592844
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,15,0.011087999989589056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,15,0.009429333110650381
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,31,0.01099733387430509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,1,0.010549332946538925
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,63,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,63,0.010853332777818045
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,127,0.010533332824707031
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,127,0.00903466654320558
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,255,0.009679999823371569
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,255,0.010186666622757912
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,511,0.010949333508809408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,32767,0.019152000546455383
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,511,0.010949333508809408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,2047,0.01110400011142095
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,1023,0.01331199953953425
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,2047,0.013381333400805792
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,31,0.011002667248249054
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,4095,0.030016000072161358
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,1,0.010778666784365972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,8191,0.04956266780694326
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,4095,0.017269333203633625
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,1,0.009237333511312803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,3,0.00938666673998038
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,7,0.009232000137368837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,7,0.010933333386977514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,15,0.009242666885256767
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,1023,0.012810666114091873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,31,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,63,0.009381333366036415
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,8191,0.032416000962257385
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,127,0.010058666889866194
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,3,0.010048000141978264
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,127,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,7,0.009290666629870733
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,15,0.009312000125646591
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,255,0.009312000125646591
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,511,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,2047,0.015109332899252573
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,1023,0.013072000195582708
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,2047,0.015018666783968607
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,2047,0.02959466725587845
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,63,0.008938666433095932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,4095,0.03035733352104823
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,255,0.00938666673998038
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,511,0.010981333752473196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,1,0.011141333729028702
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,1,0.011141333729028702
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,31,0.011071999867757162
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,3,0.011322667201360067
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,3,0.011120000233252844
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,7,0.011007999380429586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,4095,0.047775998711586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,31,0.011391999820868174
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,31,0.011109333485364914
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,8191,0.08496000369389851
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,63,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,127,0.011157333850860596
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,127,0.01090666651725769
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,1023,0.011610666910807291
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,7,0.010938666760921478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,255,0.011007999380429586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,15,0.01139733319481214
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,15,0.011007999380429586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,511,0.01302933320403099
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,1023,0.029391999046007793
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,1023,0.015317333241303762
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,8191,0.049546668926874794
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,63,0.01097600037852923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,4095,0.08438400427500407
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,4095,0.04974400003751119
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,8191,0.15731199582417807
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,3,0.009365333244204521
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,255,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,511,0.013317332913478216
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,7,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,7,0.008858666444818178
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,2047,0.04807466765244802
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,15,0.009205333267649015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,2047,0.031221332649389904
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,31,0.009663999701539675
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,31,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,63,0.00996800015370051
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,127,0.009077333534757296
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,127,0.009349333122372627
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,255,0.008842666943868002
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,511,0.011007999380429586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,3,0.009568000212311745
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,511,0.010890666395425797
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,1023,0.011296000331640244
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,15,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,1023,0.01109333336353302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,2047,0.010992000500361124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,63,0.009072000160813332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,4095,0.011136000355084738
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,1,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,8191,0.012874666601419449
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,16383,0.018021332720915478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,16383,0.01725333308180173
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,32767,0.01923199991385142
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,2047,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,32767,0.019205333044131596
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,65535,0.03691199918588003
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,65535,0.021242665747801464
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,4095,0.010928000013033548
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,1,0.00915733352303505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,3,0.00878399983048439
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,8191,0.012997332960367203
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,7,0.008938666433095932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,15,0.008949333180983862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,15,0.009045333291093508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,31,0.009194666519761086
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,63,0.008922666932145754
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,63,0.009008000294367472
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,127,0.009162666896979014
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,1,0.009082666908701261
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,3,0.009098666409651438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,255,0.009984000275532404
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,511,0.010874666273593903
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,511,0.010389333590865135
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,31,0.00890666681031386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,1023,0.011055999745925268
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,1023,0.011071999867757162
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,2047,0.011034666250149408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,2047,0.010805333654085795
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,4095,0.011109333485364914
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,4095,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,8191,0.014192000031471252
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,255,0.00891733355820179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,7,0.00898133342464765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,32767,0.03349866718053818
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,32767,0.019173332800467808
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,65535,0.05198933184146881
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,65535,0.03612799942493439
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,1,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,127,0.009162666896979014
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,8191,0.013381333400805792
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,255,0.008869333192706108
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,3,0.009029333169261614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,7,0.008949333180983862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,7,0.00919999989370505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,15,0.008879999940594038
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,31,0.008869333192706108
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,31,0.009237333511312803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,63,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,1,0.009093333035707474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,63,0.009493333597977957
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,127,0.009045333291093508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,127,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,255,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,15,0.009205333267649015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,511,0.01099733387430509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,511,0.010901333143313726
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,1023,0.011002667248249054
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,1023,0.010847999403874079
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,2047,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,8191,0.08732799688975017
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,4095,0.013173333058754602
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,3,0.009296000003814697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,4095,0.013189333180586496
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,255,0.008938666433095932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,8191,0.015119999647140503
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,16383,0.03143466760714849
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,16383,0.017338667064905167
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,32767,0.05007466673851013
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,32767,0.03332799921433131
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,65535,0.08533333738644917
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,65535,0.05209066470464071
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,1,0.010938666760921478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,2047,0.010928000013033548
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,3,0.011136000355084738
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,16383,0.017194667210181553
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,3,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,7,0.011066666493813196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,8191,0.015557333827018738
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,15,0.011077333241701126
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,15,0.011066666493813196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,31,0.010938666760921478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,63,0.011125333607196808
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,63,0.010960000256697336
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,1,0.012191999703645706
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,127,0.010992000500361124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,1,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,255,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,255,0.01097600037852923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,511,0.010992000500361124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,31,0.01091733326514562
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,2047,0.030031998952229817
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,127,0.01108266661564509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,2047,0.01714133347074191
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,4095,0.04836800197760264
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,4095,0.03143999973932902
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,1,0.01192533348997434
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,511,0.012522666404644648
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,1023,0.013455999394257864
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,3,0.011221333096424738
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,3,0.010928000013033548
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,7,0.011242666592200598
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,7,0.011071999867757162
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,15,0.012058666596810022
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,16383,0.0161013330022494
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,31,0.011349332829316458
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,31,0.010938666760921478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,7,0.010949333508809408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,63,0.012357333054145178
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,1023,0.013045333325862885
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,127,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,1,0.011226666470368704
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,255,0.0116799995303154
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,255,0.01089599976936976
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,511,0.013621332744757334
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,15,0.01108266661564509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,1023,0.029909332593282063
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,4095,0.08638933300971985
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,2047,0.04948799808820089
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,127,0.011391999820868174
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,1,0.014746667196353277
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,1,0.013007999708255133
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,3,0.01482133318980535
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,511,0.01321600005030632
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,3,0.01301866645614306
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,1023,0.015989333391189575
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,7,0.014752000570297241
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,15,0.014458666245142618
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,15,0.013167999684810638
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,4095,0.04982399940490723
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,31,0.01309866706530253
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,63,0.015029333531856537
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,127,0.013349333157142004
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,255,0.015178666760524115
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,255,0.013167999684810638
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,7,0.013072000195582708
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,511,0.029733332494894665
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,511,0.01695466662446658
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,63,0.013317332913478216
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,1023,0.03153600047032038
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,127,0.014352000008026758
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,2047,0.08533866206804912
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,2047,0.050255998969078064
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,4095,0.08798399567604065
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,4095,0.15851199626922607
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,2047,0.031530665854612984
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,1023,0.048901334404945374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,3,0.012863999853531519
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,3,0.013034666577974955
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,7,0.013056000073750814
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,7,0.013125333935022354
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,15,0.012986666212479273
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,1,0.013082666943470636
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,1,0.01314666618903478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,63,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,31,0.013093333691358566
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,63,0.012991999586423239
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,127,0.013034666577974955
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,255,0.013061333447694778
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,15,0.013023999830087027
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,31,0.015157333264748255
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,511,0.015141333142916361
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,63,0.013013333082199097
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,1023,0.031109333038330078
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,127,0.012975999464591345
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,1023,0.018239999810854595
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,1,0.015082667271296183
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,1,0.01301866645614306
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,255,0.01310933381319046
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,3,0.013157332936922709
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,3,0.014138666292031607
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,31,0.016858667135238647
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,7,0.015178666760524115
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,15,0.015114666273196539
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,15,0.013002666334311167
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,31,0.014325333138306936
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,31,0.012997332960367203
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,63,0.014357333381970724
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,63,0.013082666943470636
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,127,0.015072000523408255
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,127,0.013679999858140945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,255,0.014981333166360855
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,255,0.013082666943470636
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,511,0.029834667841593426
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,511,0.01714133347074191
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,7,0.013408000270525614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,1023,0.049695998430252075
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,1,0.01931200052301089
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,3,0.01922133316596349
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,3,0.01714666684468587
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,7,0.017445333302021027
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,511,0.015173333386580149
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,15,0.01709866647919019
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,31,0.019226666539907455
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,1023,0.03164266546567281
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,31,0.01709866647919019
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,1,0.017103999853134155
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,63,0.019237333287795384
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,63,0.01781333362062772
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,127,0.01953599974513054
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,127,0.017263999829689663
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,255,0.017210666090250015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,511,0.049312000473340355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,511,0.03152533372243246
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,7,0.01926400015751521
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,1023,0.050613333781560264
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,15,0.01918399954835574
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,255,0.03047466774781545
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,1,0.017152000218629837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,1,0.015061333775520325
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,3,0.017184000462293625
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,7,0.017263999829689663
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,3,0.015194666882356008
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,7,0.015082667271296183
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,15,0.0170666662355264
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,15,0.015189333508412043
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,31,0.017184000462293625
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,31,0.015237333873907724
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,63,0.01710933322707812
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,63,0.015135999768972397
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,127,0.01721599946419398
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,1023,0.08462400237719218
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,127,0.015306666493415833
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,255,0.014997333288192749
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,511,0.03140799949566523
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,1,0.019237333287795384
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,1,0.017242666333913803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,3,0.01770666614174843
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,3,0.01913600042462349
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,7,0.019098666807015736
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,255,0.01721599946419398
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,15,0.019173332800467808
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,511,0.019237333287795384
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,31,0.01748266691962878
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,63,0.017269333203633625
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,127,0.019274666905403137
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,7,0.01728533332546552
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,15,0.017397332936525345
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,127,0.01720000058412552
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,31,0.019093333433071773
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,63,0.019130667050679524
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,511,0.04994133114814758
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,1,0.027376001079877216
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,3,0.03149333347876867
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,3,0.027386667827765148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,255,0.017386666188637417
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,7,0.027429332335789997
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,511,0.03249600032965342
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,15,0.03143999973932902
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,15,0.02736533433198929
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,255,0.031008000175158184
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,31,0.031541332602500916
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,31,0.027466667195161183
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,7,0.03151999910672506
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,1,0.032618666688601174
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,63,0.02787200113137563
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,63,0.032218667368094124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,1,0.008949333180983862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,255,0.05095999936262766
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,127,0.035829332967599235
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,1,0.009658666948477427
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,255,0.035562666753927864
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,7,0.009066666786869368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,3,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,15,0.00919999989370505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,15,0.00919999989370505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,31,0.009029333169261614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,63,0.008949333180983862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,511,0.052928000688552856
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,63,0.009408000235756239
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,127,0.009279999881982803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,3,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,127,0.01044800008336703
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,511,0.010885333021481832
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,7,0.009589333087205887
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,1023,0.011130666981140772
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,2047,0.011050666371981302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,2047,0.011168000598748526
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,127,0.029359998802344005
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,255,0.009354666496316591
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,4095,0.01116266722480456
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,8191,0.015061333775520325
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,8191,0.015173333386580149
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,511,0.010938666760921478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,16383,0.01714666684468587
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,32767,0.033626665671666466
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,31,0.00898133342464765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,65535,0.053258667389551796
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,511,0.0874826709429423
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,65535,0.035760000348091125
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,4095,0.011120000233252844
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,3,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,3,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,255,0.009029333169261614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,1,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,16383,0.018101333330074947
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,1023,0.011194666226704916
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,15,0.009285333255926767
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,31,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,31,0.009194666519761086
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,63,0.009056000038981438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,127,0.008805333326260248
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,127,0.008901333436369896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,7,0.009056000038981438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,255,0.010933333386977514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,15,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,255,0.009050666665037474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,511,0.00983466642598311
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,511,0.010858666151762009
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,1023,0.010938666760921478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,2047,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,2047,0.010922666639089584
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,63,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,4095,0.012954667210578918
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,8191,0.015840000162522
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,8191,0.015077333897352219
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,16383,0.01725333308180173
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,32767,0.050848002235094704
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,7,0.009088000282645226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,65535,0.08676266670227051
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,65535,0.052005335688591
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,1023,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,1,0.009103999783595404
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,1,0.009056000038981438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,3,0.009082666908701261
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,7,0.009253333633144697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,7,0.00903466654320558
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,1,0.009066666786869368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,16383,0.031770666440327965
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,31,0.009370666618148485
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,31,0.009568000212311745
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,32767,0.03435733417669932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,63,0.009205333267649015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,32767,0.020527999848127365
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,127,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,3,0.009039999917149544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,255,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,15,0.009152000149091085
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,15,0.009103999783595404
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,511,0.011071999867757162
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,1023,0.010879999647537867
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,63,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,2047,0.01109333336353302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,4095,0.013712000101804733
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,127,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,8191,0.029802667597929638
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,255,0.00891733355820179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,16383,0.04785599807898203
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,4095,0.013104000439246496
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,16383,0.03155199935038885
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,32767,0.08437333504358928
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,32767,0.05014933149019877
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,1023,0.011050666371981302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,65535,0.1567253371079763
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,2047,0.013061333447694778
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,4095,0.01309866706530253
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,8191,0.015728000551462173
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,511,0.011109333485364914
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,65535,0.08684800068537395
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,1,0.025306666890780132
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,3,0.025392000873883564
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,3,0.021295999487241108
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,1,0.021754667162895203
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,15,0.02550933261712392
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,15,0.021205333371957142
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,31,0.025279998779296875
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,7,0.02164799968401591
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,7,0.025397333006064098
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,31,0.02128533273935318
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,63,0.025205334027608235
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,127,0.025248001019159954
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,63,0.021013334393501282
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,127,0.020581333587567013
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,1,0.027658666173617046
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,255,0.021146667500336964
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,3,0.031514666974544525
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,1,0.03145066648721695
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,7,0.031530665854612984
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,15,0.031530665854612984
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,7,0.027445333699385326
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,15,0.027701333165168762
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,31,0.03154666721820831
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,255,0.03173866619666418
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,31,0.028378665447235107
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,63,0.031583999594052635
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,63,0.027930667002995808
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,3,0.027818667391935985
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,127,0.02874133239189784
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,255,0.036831999818483986
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,1,0.04794666667779287
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,127,0.035605333745479584
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,3,0.05801600217819214
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,3,0.04790399968624115
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,255,0.05184000233809153
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,7,0.05809600154558817
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,1,0.05818133552869161
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,7,0.04814933240413666
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,15,0.058378666639328
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,15,0.0479360024134318
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,31,0.057989334066708885
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,31,0.04781333108743032
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,63,0.0580213318268458
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,63,0.04796266555786133
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,127,0.06041066845258077
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,127,0.05197866757710775
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,255,0.09296533465385437
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,255,0.06136533121267954
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,1,0.04164800047874451
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,1,0.03151999910672506
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,3,0.041696002086003624
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,3,0.03266666581233343
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,7,0.041589332123597465
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,7,0.03278400003910065
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,15,0.0418453315893809
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,31,0.041759997606277466
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,15,0.0317546675602595
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,31,0.03143999973932902
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,63,0.041573333243529
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,63,0.031530665854612984
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,127,0.04381333291530609
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,127,0.033514666060606636
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,1,0.05606399973233541
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,1,0.04811733464399973
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,3,0.055999999245007835
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,3,0.04750399788220724
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,7,0.05600533386071523
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,7,0.04790933430194855
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,15,0.056373332937558494
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,15,0.048165331284205117
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,31,0.056320001681645714
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,31,0.04717333118120829
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,63,0.056032001972198486
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,63,0.04714133342107137
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,1,0.10407466689745586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,127,0.05057600140571594
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,127,0.060122668743133545
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,3,0.10583999752998352
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,3,0.09319999814033508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,7,0.1051093339920044
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,7,0.09297066926956177
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,15,0.10429333647092183
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,15,0.09295466542243958
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,1,0.009109333157539368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,1,0.009050666665037474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,3,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,3,0.009109333157539368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,7,0.009130666653315226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,31,0.10517332951227824
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,7,0.009082666908701261
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,31,0.09178666273752849
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,15,0.009066666786869368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,15,0.010010666524370512
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,31,0.009045333291093508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,1,0.09316800038019817
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,63,0.0100426667680343
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,127,0.010015999898314476
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,255,0.009637333452701569
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,63,0.10540800293286641
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,255,0.009343999748428663
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,511,0.011055999745925268
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,511,0.011146667102972666
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,63,0.09290132919947307
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,1023,0.011050666371981302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,63,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,2047,0.011173332730929056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,2047,0.011168000598748526
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,8191,0.017082666357358296
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,8191,0.017045332739750545
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,127,0.09464533130327861
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,16383,0.03346133232116699
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,127,0.1104213297367096
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,1023,0.011391999820868174
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,1,0.008885333314538002
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,31,0.010037333394090334
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,1,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,3,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,16383,0.01921066641807556
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,32767,0.036389333506425224
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,3,0.009029333169261614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,15,0.008912000184257826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,31,0.008922666932145754
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,31,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,32767,0.05152533451716105
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,63,0.008879999940594038
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,127,0.009082666908701261
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,4095,0.014736000448465347
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,127,0.009328000247478485
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,7,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,7,0.008879999940594038
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,255,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,511,0.011066666493813196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,511,0.01109333336353302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,1023,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,2047,0.011130666981140772
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,4095,0.013728000223636627
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,63,0.008879999940594038
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,4095,0.012853333105643591
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,4095,0.013408000270525614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,8191,0.029296000798543293
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,8191,0.01526933287580808
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,16383,0.04775999983151754
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,15,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,16383,0.03136533250411352
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,1023,0.011002667248249054
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,1,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,2047,0.01097600037852923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,3,0.009258666386206945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,7,0.00898133342464765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,7,0.00926399976015091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,15,0.00898133342464765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,255,0.009008000294367472
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,15,0.009258666386206945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,31,0.009194666519761086
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,31,0.009583999713261923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,1,0.009029333169261614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,63,0.009455999980370203
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,127,0.00966933307548364
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,127,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,255,0.009205333267649015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,255,0.009946666657924652
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,511,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,511,0.011173332730929056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,127,0.00943999985853831
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,1023,0.011429333438475927
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,2047,0.013173333058754602
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,2047,0.013717333475748697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,4095,0.029370665550231934
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,4095,0.015184000134468079
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,32767,0.08494933446248372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,8191,0.03141866624355316
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,16383,0.08370666702588399
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,63,0.00890666681031386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,32767,0.15643200278282166
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,1,0.008949333180983862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,1,0.009850666547815004
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,3,0.008832000195980072
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,3,0.00890666681031386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,7,0.00949866697192192
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,7,0.00890666681031386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,15,0.009429333110650381
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,15,0.009152000149091085
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,8191,0.04820266862710317
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,31,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,16383,0.04985600213209788
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,3,0.009183999771873156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,63,0.009322666873534521
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,127,0.00915733352303505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,127,0.009594666461149851
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,255,0.00903466654320558
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,255,0.009109333157539368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,1023,0.011098666737476984
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,511,0.010911999891201654
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,1023,0.011264000087976456
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,2047,0.012805332740147909
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,2047,0.011253333340088526
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,31,0.008933333059151968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,4095,0.015061333775520325
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,63,0.008885333314538002
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,8191,0.02943466603755951
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,8191,0.01532799998919169
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,32767,0.08619733651479085
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,16383,0.04783466458320618
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,16383,0.03141866624355316
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,32767,0.08635733524958293
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,1,0.009173333023985228
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,1,0.010661333799362183
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,3,0.009568000212311745
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,3,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,7,0.010490667074918747
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,7,0.009663999701539675
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,15,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,15,0.009178666397929192
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,31,0.00892800030608972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,31,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,63,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,63,0.008938666433095932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,127,0.009029333169261614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,255,0.009237333511312803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,32767,0.05034666756788889
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,255,0.00943999985853831
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,1023,0.011066666493813196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,1023,0.010938666760921478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,1023,0.011034666250149408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,2047,0.012981332838535309
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,4095,0.013045333325862885
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,127,0.009610666582981745
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,4095,0.01524266724785169
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,8191,0.0488319993019104
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,511,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,511,0.011194666226704916
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,16383,0.08572799960772197
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,511,0.010933333386977514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,32767,0.08692800005276997
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,1,0.010773333410422007
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,1,0.010992000500361124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,3,0.011125333607196808
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,4095,0.029648000995318096
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,3,0.010757333288590113
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,8191,0.03130666663249334
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,7,0.010442666709423065
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,15,0.009077333534757296
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,16383,0.04984533290068308
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,31,0.009045333291093508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,32767,0.15847466389338175
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,63,0.009408000235756239
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,2047,0.013130666067202887
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,127,0.009397333487868309
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,127,0.011136000355084738
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,255,0.01091733326514562
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,7,0.00926399976015091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,511,0.010981333752473196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,15,0.010938666760921478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,1023,0.013104000439246496
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,2047,0.02942933390537898
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,2047,0.015168000012636185
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,4095,0.047770669062932335
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,8191,0.08334400256474812
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,255,0.00898133342464765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,511,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,8191,0.04818133513132731
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,1023,0.013151999562978745
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,16383,0.0860053300857544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,31,0.009189333145817121
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,32767,0.30010666449864704
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,1,0.010949333508809408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,1,0.011535999675591787
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,32767,0.16075733304023743
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,3,0.011343999455372492
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,3,0.010890666395425797
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,7,0.011594666788975397
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,15,0.011173332730929056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,31,0.01129066695769628
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,31,0.01109333336353302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,16383,0.1560533344745636
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,127,0.011333333949247995
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,127,0.011098666737476984
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,255,0.01163200040658315
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,4095,0.030837332208951313
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,7,0.01099733387430509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,511,0.015103999525308609
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,511,0.013034666577974955
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,15,0.010826667149861654
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,1023,0.01573866605758667
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,63,0.011215999722480774
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,2047,0.030821333328882854
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,4095,0.049866666396458946
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,4095,0.08487466971079509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,8191,0.1577173372109731
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,8191,0.08610666791598003
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,63,0.01097600037852923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,255,0.01110400011142095
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,16383,0.30313066641489667
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,16383,0.16146666804949442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,1023,0.02935466667016347
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,1,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,32767,0.3090239961942037
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,1,0.009445333232482275
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,7,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,15,0.009269333134094873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,7,0.008896000062425932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,31,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,31,0.009061333412925402
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,63,0.009008000294367472
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,127,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,127,0.009039999917149544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,255,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,32767,0.5983893473943075
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,2047,0.04780266682306925
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,63,0.011237333218256632
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,3,0.009269333134094873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,511,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,15,0.00891733355820179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,1023,0.010960000256697336
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,2047,0.010682666053374609
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,4095,0.010901333143313726
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,4095,0.01080000028014183
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,63,0.00891733355820179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,8191,0.01119999960064888
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,16383,0.015135999768972397
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,32767,0.01916266605257988
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,255,0.009072000160813332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,3,0.009008000294367472
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,65535,0.027082666754722595
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,65535,0.026367999613285065
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,511,0.009269333134094873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,131071,0.03149333347876867
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,1,0.009088000282645226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,131071,0.030495998760064442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,1,0.008949333180983862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,8191,0.011333333949247995
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,3,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,7,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,15,0.009029333169261614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,15,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,32767,0.01912533367673556
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,31,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,1023,0.010960000256697336
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,63,0.00898133342464765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,2047,0.010805333654085795
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,63,0.008938666433095932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,127,0.009056000038981438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,255,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,3,0.008879999940594038
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,255,0.00890666681031386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,511,0.010847999403874079
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,7,0.009568000212311745
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,1023,0.011168000598748526
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,31,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,2047,0.011087999989589056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,4095,0.011242666592200598
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,8191,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,8191,0.011071999867757162
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,16383,0.015168000012636185
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,16383,0.015311999867359797
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,16383,0.014869333555301031
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,511,0.010874666273593903
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,32767,0.019141333798567455
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,1023,0.011055999745925268
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,131071,0.040250666439533234
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,4095,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,1,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,3,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,3,0.009258666386206945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,127,0.00890666681031386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,7,0.009237333511312803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,32767,0.019365333020687103
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,15,0.009359999870260557
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,65535,0.022917332748572033
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,2047,0.011146667102972666
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,1,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,63,0.009045333291093508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,127,0.009039999917149544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,127,0.008837333569924036
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,7,0.009173333023985228
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,255,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,15,0.009098666409651438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,31,0.008922666932145754
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,63,0.009082666908701261
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,1023,0.011120000233252844
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,2047,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,2047,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,255,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,4095,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,8191,0.011322667201360067
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,511,0.009578666960199675
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,8191,0.011087999989589056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,1023,0.011039999624093374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,16383,0.015135999768972397
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,32767,0.01942933350801468
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,32767,0.019237333287795384
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,65535,0.03548266738653183
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,65535,0.019952000429232914
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,131071,0.05460800230503082
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,4095,0.011087999989589056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,131071,0.03769599894682566
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,1,0.00916800027092298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,65535,0.02330133318901062
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,1,0.009338666374484697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,3,0.009119999905427298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,3,0.009008000294367472
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,7,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,15,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,15,0.009141333401203156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,31,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,31,0.009248000259200731
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,63,0.009136000027259191
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,127,0.009365333244204521
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,127,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,255,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,511,0.010954666882753372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,511,0.01109333336353302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,31,0.009381333366036415
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,16383,0.015423999478419622
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,1023,0.011605333536863327
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,63,0.00903466654320558
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,2047,0.011253333340088526
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,2047,0.011109333485364914
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,4095,0.010981333752473196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,255,0.009029333169261614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,8191,0.014127999544143677
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,16383,0.01725333308180173
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,131071,0.02359466751416524
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,1023,0.010954666882753372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,32767,0.019296000401178997
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,65535,0.05178666611512502
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,131071,0.08705066641171773
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,8191,0.01331199953953425
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,131071,0.05302399893601736
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,1,0.009082666908701261
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,1,0.008826666822036108
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,3,0.00890666681031386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,3,0.00890666681031386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,511,0.009269333134094873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,7,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,15,0.00921066664159298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,15,0.009061333412925402
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,65535,0.035487999518712364
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,31,0.009189333145817121
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,31,0.009205333267649015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,63,0.009098666409651438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,4095,0.01137599969903628
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,127,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,16383,0.01589866727590561
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,255,0.009514666472872099
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,7,0.009514666472872099
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,7,0.009050666665037474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,511,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,1023,0.010954666882753372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,2047,0.010933333386977514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,2047,0.01090666651725769
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,63,0.009232000137368837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,4095,0.011173332730929056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,4095,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,8191,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,8191,0.011050666371981302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,16383,0.01526933287580808
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,32767,0.019258666783571243
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,32767,0.03356266766786575
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,511,0.009178666397929192
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,65535,0.02142400046189626
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,131071,0.039503999054431915
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,131071,0.023418667415777843
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,1,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,1,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,3,0.00891733355820179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,127,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,7,0.008816000074148178
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,7,0.009114666531483332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,16383,0.015461333096027374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,15,0.009066666786869368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,31,0.008912000184257826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,1023,0.01097600037852923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,63,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,127,0.008933333059151968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,127,0.00898133342464765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,3,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,255,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,511,0.010949333508809408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,15,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,32767,0.01939733326435089
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,1023,0.010960000256697336
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,63,0.009103999783595404
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,2047,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,2047,0.01099733387430509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,4095,0.010954666882753372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,4095,0.011183999478816986
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,8191,0.01089599976936976
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,255,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,16383,0.015103999525308609
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,255,0.00938666673998038
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,32767,0.019333332777023315
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,65535,0.0233599990606308
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,65535,0.03621866554021835
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,65535,0.0195573332409064
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,131071,0.05465066432952881
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,8191,0.01116266722480456
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,1,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,16383,0.015376000354687372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,3,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,3,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,7,0.009056000038981438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,31,0.008869333192706108
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,15,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,32767,0.017509333789348602
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,31,0.009050666665037474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,63,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,131071,0.03868799904982249
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,127,0.00915733352303505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,1,0.009253333633144697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,127,0.00903466654320558
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,255,0.009119999905427298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,511,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,511,0.011157333850860596
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,7,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,15,0.009114666531483332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,1023,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,1023,0.010954666882753372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,1023,0.011034666250149408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,2047,0.011136000355084738
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,4095,0.010981333752473196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,63,0.009039999917149544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,8191,0.013072000195582708
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,16383,0.017258666455745697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,16383,0.015754666179418564
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,32767,0.03393599887688955
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,32767,0.019130667050679524
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,65535,0.05197333296140035
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,255,0.009173333023985228
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,65535,0.035573333501815796
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,511,0.01110400011142095
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,131071,0.05429333448410034
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,31,0.009258666386206945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,1,0.009338666374484697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,4095,0.010901333143313726
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,3,0.009925333162148794
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,8191,0.01310933381319046
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,7,0.00922133338948091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,15,0.009056000038981438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,15,0.009162666896979014
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,31,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,31,0.00927466650803884
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,63,0.009039999917149544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,63,0.009375999992092451
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,131071,0.08853333195050557
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,127,0.009072000160813332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,2047,0.011098666737476984
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,255,0.009050666665037474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,511,0.010901333143313726
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,1023,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,1023,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,2047,0.011034666250149408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,2047,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,1,0.009989333028594652
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,4095,0.013189333180586496
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,7,0.008949333180983862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,16383,0.03169599920511246
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,16383,0.017082666357358296
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,32767,0.04995200037956238
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,4095,0.013002666334311167
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,255,0.009008000294367472
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,3,0.008874666566650072
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,65535,0.08596266309420268
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,65535,0.051701332132021584
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,511,0.00972800018886725
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,131071,0.0883679986000061
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,1,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,1,0.009370666618148485
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,3,0.009269333134094873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,3,0.009072000160813332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,7,0.009237333511312803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,15,0.009082666908701261
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,32767,0.03319466610749563
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,15,0.009509333098928133
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,31,0.008885333314538002
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,63,0.00921066664159298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,8191,0.01714666684468587
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,131071,0.15737600127855936
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,127,0.0099093330403169
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,7,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,255,0.009029333169261614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,255,0.009226666763424873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,511,0.010863999525705973
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,127,0.009162666896979014
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,1023,0.011039999624093374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,2047,0.013354666531085968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,31,0.009253333633144697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,8191,0.015109332899252573
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,2047,0.013061333447694778
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,8191,0.04855466882387797
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,127,0.00903466654320558
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,8191,0.031504000226656594
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,4095,0.01579733317097028
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,16383,0.08546132842699687
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,16383,0.04979733129342397
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,1,0.010575999816258749
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,511,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,3,0.010922666639089584
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,7,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,7,0.009232000137368837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,15,0.00927466650803884
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,63,0.009237333511312803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,15,0.010298666854699453
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,31,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,31,0.00903466654320558
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,63,0.010645333677530289
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,4095,0.030447999636332195
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,127,0.00919999989370505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,1,0.010816000401973724
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,255,0.01128000020980835
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,1023,0.011247999966144562
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,511,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,1023,0.013269333789745966
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,2047,0.02938666691382726
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,2047,0.015168000012636185
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,63,0.009946666657924652
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,4095,0.047839999198913574
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,8191,0.08459200461705525
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,8191,0.04938666522502899
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,255,0.009632000078757605
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,3,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,16383,0.08520533641179402
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,1,0.011226666470368704
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,3,0.011861333002646765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,3,0.01139733319481214
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,7,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,127,0.009301333377758661
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,7,0.011296000331640244
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,4095,0.03148266673088074
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,15,0.011055999745925268
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,31,0.011882666498422623
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,511,0.011136000355084738
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,63,0.011391999820868174
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,16383,0.15908799568812051
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,63,0.011407999942700068
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,1,0.011087999989589056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,127,0.011002667248249054
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,255,0.011653333902359009
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,255,0.011066666493813196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,511,0.014389333625634512
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,511,0.013199999928474426
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,1023,0.029365333418051403
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,15,0.011413333316644033
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,2047,0.048485333720842995
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,4095,0.08469866712888081
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,4095,0.04993600149949392
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,127,0.011114666859308878
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,8191,0.1567626694838206
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,8191,0.08778132994969685
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,16383,0.30216000477472943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,1,0.015178666760524115
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,16383,0.16133333245913187
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,3,0.015072000523408255
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,1,0.013088000317414602
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,31,0.01181866725285848
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,1023,0.011301333705584208
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,7,0.012975999464591345
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,7,0.014874666929244995
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,15,0.01431999976436297
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,31,0.01302933320403099
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,63,0.015103999525308609
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,127,0.015413332730531693
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,3,0.013050666699806849
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,127,0.013424000392357508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,255,0.01303999995191892
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,255,0.015520000209410986
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,511,0.030581332743167877
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,31,0.015184000134468079
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,1023,0.04930133124192556
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,63,0.013151999562978745
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,2047,0.08469866712888081
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,2047,0.05022400120894114
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,2047,0.03127466638882955
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,4095,0.15786133209864298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,8191,0.30449599027633667
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,8191,0.1588746706644694
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,15,0.012944000462690989
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,16383,0.5973493258158366
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,1,0.008938666433095932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,1,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,3,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,1023,0.031744000812371574
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,7,0.009232000137368837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,15,0.00919999989370505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,15,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,31,0.009141333401203156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,63,0.008858666444818178
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,63,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,511,0.017029333859682083
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,127,0.009114666531483332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,1023,0.015141333142916361
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,16383,0.30456533034642536
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,255,0.009381333366036415
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,4095,0.08669333656628926
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,511,0.010981333752473196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,1023,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,1023,0.01116266722480456
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,2047,0.010757333288590113
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,127,0.009050666665037474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,4095,0.011066666493813196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,4095,0.011007999380429586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,8191,0.011242666592200598
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,3,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,8191,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,7,0.008879999940594038
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,16383,0.01534933348496755
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,32767,0.017407999684413273
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,511,0.01097600037852923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,65535,0.01952533299724261
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,31,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,131071,0.03585066646337509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,2047,0.011071999867757162
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,1,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,3,0.00921066664159298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,7,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,255,0.009008000294367472
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,7,0.009248000259200731
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,32767,0.01930133377512296
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,15,0.008863999818762144
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,31,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,131071,0.054042667150497437
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,1,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,63,0.009194666519761086
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,127,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,255,0.009008000294367472
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,255,0.00892800030608972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,511,0.010933333386977514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,16383,0.015520000209410986
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,15,0.009056000038981438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,511,0.009973333527644476
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,1023,0.011071999867757162
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,2047,0.010698666175206503
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,63,0.009397333487868309
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,3,0.009008000294367472
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,4095,0.01091733326514562
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,8191,0.013408000270525614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,8191,0.013066666821638743
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,16383,0.01708799973130226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,16383,0.015087999403476715
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,65535,0.035461333890755974
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,31,0.009189333145817121
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,32767,0.019039999693632126
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,2047,0.011071999867757162
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,65535,0.052144000927607216
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,4095,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,131071,0.08875200152397156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,1,0.00898133342464765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,1,0.009248000259200731
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,3,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,3,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,7,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,7,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,15,0.009253333633144697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,1023,0.011007999380429586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,31,0.009082666908701261
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,31,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,63,0.009258666386206945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,65535,0.03393599887688955
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,127,0.009178666397929192
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,127,0.009322666873534521
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,131071,0.054229333996772766
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,255,0.00914666677514712
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,255,0.00972800018886725
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,1023,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,1023,0.011237333218256632
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,2047,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,2047,0.011002667248249054
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,4095,0.012986666212479273
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,8191,0.017082666357358296
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,63,0.009183999771873156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,8191,0.014917333920796713
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,127,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,16383,0.017386666188637417
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,32767,0.04993066688378652
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,511,0.010933333386977514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,32767,0.033488000432650246
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,65535,0.08577066659927368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,65535,0.05176533261934916
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,15,0.009088000282645226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,131071,0.1590559979279836
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,131071,0.08891200025876363
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,4095,0.013317332913478216
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,3,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,7,0.009258666386206945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,7,0.009114666531483332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,15,0.01033599985142549
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,16383,0.03239466746648153
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,511,0.010709332923094431
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,31,0.009765333185593287
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,63,0.00973866693675518
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,1,0.009152000149091085
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,127,0.009205333267649015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,32767,0.03358400116364161
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,3,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,127,0.009269333134094873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,255,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,511,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,511,0.010901333143313726
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,1023,0.01091733326514562
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,1023,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,2047,0.010933333386977514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,31,0.009178666397929192
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,4095,0.013056000073750814
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,63,0.009178666397929192
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,8191,0.03029866764942805
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,1,0.00926399976015091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,16383,0.03183999905983607
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,255,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,32767,0.0483893354733785
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,15,0.008938666433095932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,65535,0.15632533033688864
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,2047,0.013034666577974955
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,4095,0.01340266689658165
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,131071,0.1588106652100881
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,1,0.011034666250149408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,8191,0.015018666783968607
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,3,0.010965333630641302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,16383,0.0480373352766037
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,3,0.010879999647537867
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,7,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,32767,0.08338666955629985
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,7,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,15,0.009578666960199675
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,31,0.009957333405812582
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,63,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,63,0.009503999724984169
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,127,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,1,0.009674666449427605
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,127,0.010773333410422007
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,255,0.010378666842977205
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,255,0.009765333185593287
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,511,0.01097600037852923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,511,0.011114666859308878
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,15,0.011071999867757162
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,1023,0.012410666793584824
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,2047,0.0295413335164388
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,2047,0.015216000378131866
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,131071,0.3004053235054016
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,4095,0.04818666477998098
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,4095,0.031157332162062328
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,8191,0.08470933636029561
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,8191,0.04948799808820089
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,1023,0.013013333082199097
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,3,0.011109333485364914
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,3,0.01090666651725769
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,65535,0.08513599634170532
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,7,0.011007999380429586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,15,0.010965333630641302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,15,0.011296000331640244
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,31,0.010869332899649939
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,63,0.011253333340088526
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,63,0.011077333241701126
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,1,0.012261333564917246
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,31,0.010938666760921478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,127,0.010933333386977514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,7,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,255,0.01091733326514562
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,511,0.013786666095256805
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,31,0.010837333897749582
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,1023,0.029322666426499683
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,1023,0.015392000476519266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,2047,0.048810665806134544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,2047,0.031680000325044
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,4095,0.05096533397833506
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,255,0.01108266661564509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,8191,0.16051733493804932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,511,0.012949333836634954
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,1,0.013429333766301474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,3,0.015072000523408255
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,7,0.013776000589132309
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,127,0.011231999844312668
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,7,0.013007999708255133
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,15,0.01488000030318896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,15,0.013248000293970108
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,31,0.015146666516860327
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,1,0.013760000467300415
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,8191,0.08878399928410848
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,63,0.015770666301250458
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,63,0.013061333447694778
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,3,0.01312000056107839
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,127,0.013290667285521826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,1,0.011312000453472137
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,255,0.012960000584522883
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,511,0.02942399928967158
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,31,0.01331199953953425
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,1023,0.048512001832326256
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,2047,0.08540266752243042
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,2047,0.05029866596062978
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,127,0.015354666858911514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,4095,0.08777067065238953
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,4095,0.15890133380889893
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,255,0.015157333264748255
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,511,0.016938666502634685
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,8191,0.3110613425572713
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,1,0.019205333044131596
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,1,0.017290666699409485
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,1023,0.03166933357715607
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,3,0.01735466718673706
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,3,0.019546666493018467
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,15,0.019205333044131596
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,31,0.019215999792019527
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,31,0.01743999992807706
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,15,0.017258666455745697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,4095,0.08611200253168742
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,63,0.01706133286158244
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,127,0.019578666736682255
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,255,0.029706666866938274
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,7,0.01716800034046173
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,255,0.017136000096797943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,511,0.0481279989083608
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,511,0.03141866624355316
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,1023,0.08489066362380981
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,63,0.01918399954835574
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,1023,0.05019199848175049
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,8191,0.16317866245905557
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,4095,0.29870933294296265
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,2047,0.08729066451390584
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,7,0.019391999890406925
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,8191,0.5861813227335612
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,8191,0.3087093234062195
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,1,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,3,0.011541333049535751
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,3,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,127,0.01718933383623759
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,7,0.011066666493813196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,4095,0.1616266667842865
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,7,0.011247999966144562
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,15,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,31,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,31,0.011215999722480774
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,63,0.011114666859308878
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,63,0.01121066634853681
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,127,0.011253333340088526
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,127,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,255,0.010949333508809408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,511,0.015311999867359797
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,15,0.011215999722480774
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,1023,0.030069333811601002
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,1,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,1023,0.015696000307798386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,2047,0.0492799977461497
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,2047,0.03151999910672506
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,1,0.015050667027632395
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,2047,0.15596266587575278
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,511,0.012997332960367203
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,3,0.015200000256299973
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,3,0.013104000439246496
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,15,0.015018666783968607
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,15,0.013797332843144735
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,31,0.015018666783968607
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,31,0.013072000195582708
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,1,0.013370666652917862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,63,0.015034666905800501
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,63,0.013397333522637686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,127,0.015061333775520325
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,255,0.01515199989080429
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,7,0.013381333400805792
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,255,0.013050666699806849
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,511,0.03046933313210805
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,511,0.017162666966517765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,255,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,7,0.015119999647140503
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,1023,0.049839998284975685
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,1023,0.03202133377393087
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,2047,0.08569066723187764
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,2047,0.05082666873931885
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,3,0.01907733331123988
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,3,0.01720000058412552
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,7,0.019487999379634857
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,15,0.017055999487638474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,127,0.013088000317414602
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,1,0.017184000462293625
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,7,0.017125333348910015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,31,0.01923199991385142
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,31,0.01720000058412552
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,63,0.01720000058412552
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,127,0.01940800001223882
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,255,0.02939733366171519
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,255,0.01728533332546552
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,1,0.019152000546455383
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,511,0.04864533245563507
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,15,0.01926933353145917
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,1023,0.08463999629020691
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,63,0.019093333433071773
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,2047,0.15677332878112793
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,127,0.017050666113694508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,1023,0.050767997900644936
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,2047,0.08874666690826416
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,3,0.029301332930723827
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,3,0.025941332181294758
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,511,0.031504000226656594
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,15,0.029359998802344005
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,7,0.02585600068171819
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,15,0.025290665527184803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,1,0.02956799914439519
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,63,0.029530666768550873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,31,0.02644266684850057
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,63,0.02571733295917511
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,127,0.033039999504884086
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,1,0.025439999997615814
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,7,0.02937600016593933
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,255,0.048810665806134544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,31,0.029477333029111225
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,1023,0.15641599893569946
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,511,0.05053333441416422
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,511,0.08536000053087871
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,2047,0.2996906638145447
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,127,0.02607999990383784
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,2047,0.1590986649195353
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,255,0.032229334115982056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,1,0.015168000012636185
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,1023,0.08660266796747844
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,1,0.013013333082199097
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,3,0.013002666334311167
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,7,0.0129120002190272
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,3,0.015141333142916361
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,15,0.013290667285521826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,15,0.014085333794355392
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,31,0.014159999787807465
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,31,0.013295999417702356
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,63,0.015157333264748255
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,63,0.013072000195582708
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,127,0.015226667126019796
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,127,0.013450667262077332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,7,0.015173333386580149
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,255,0.014325333138306936
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,255,0.012975999464591345
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,511,0.030645333230495453
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,511,0.01700266698996226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,1023,0.0497920016447703
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,1,0.01924266666173935
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,1,0.017605333278576534
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,3,0.019258666783571243
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,3,0.017370666066805523
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,7,0.0191040001809597
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,7,0.01710933322707812
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,15,0.019445333629846573
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,31,0.01717866708834966
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,63,0.019424000134070713
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,1023,0.03242666771014532
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,63,0.017194667210181553
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,127,0.019199999670187633
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,127,0.01714666684468587
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,31,0.01918399954835574
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,255,0.01757866640885671
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,511,0.04906133313973745
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,511,0.032032000521818794
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,1,0.03133866687615713
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,15,0.017050666113694508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,1023,0.08566400408744812
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,7,0.031397332747777305
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,3,0.027600000301996868
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,7,0.027562665442625683
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,255,0.0303413321574529
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,1023,0.050661335388819374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,15,0.03154666721820831
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,15,0.02739199995994568
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,31,0.03153600047032038
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,31,0.027258666853109997
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,3,0.03147733211517334
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,63,0.03138133386770884
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,127,0.03566933423280716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,1,0.02769600103298823
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,127,0.027674667537212372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,255,0.051183998584747314
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,63,0.027717334528764088
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,255,0.03585600107908249
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,511,0.08773866295814514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,1,0.05384533107280731
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,511,0.05399466554323832
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,3,0.05402133365472158
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,3,0.044250667095184326
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,1023,0.159061332543691
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,7,0.04409599800904592
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,1023,0.09010133147239685
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,7,0.05394133428732554
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,15,0.05393599967161814
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,1,0.045456002155939736
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,31,0.054586668809254967
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,63,0.054133335749308266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,15,0.044064000248909
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,127,0.05602133274078369
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,255,0.055104002356529236
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,255,0.08699733018875122
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,31,0.04577599962552389
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,1,0.009008000294367472
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,1023,0.3022293249766032
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,3,0.009066666786869368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,63,0.04569066564242045
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,1,0.00890666681031386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,511,0.15878400206565857
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,7,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,15,0.008890666688481966
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,127,0.04783466458320618
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,31,0.009119999905427298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,63,0.009183999771873156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,511,0.09059733152389526
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,127,0.00919999989370505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,63,0.009343999748428663
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,3,0.009189333145817121
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,255,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,511,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,511,0.010202666744589806
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,7,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,1023,0.010965333630641302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,31,0.008949333180983862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,2047,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,2047,0.010992000500361124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,127,0.009039999917149544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,4095,0.010911999891201654
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,8191,0.01310933381319046
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,16383,0.0170666662355264
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,255,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,16383,0.01613333324591319
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,32767,0.019434666881958645
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,65535,0.05202133456865946
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,1023,0.010981333752473196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,65535,0.035274667044480644
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,131071,0.08946667114893596
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,131071,0.053717335065205894
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,1023,0.16266666849454245
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,3,0.009754666437705358
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,8191,0.013647999614477158
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,7,0.00890666681031386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,15,0.00890666681031386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,15,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,31,0.008896000062425932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,31,0.00891733355820179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,3,0.008922666932145754
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,63,0.00891733355820179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,63,0.009061333412925402
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,4095,0.011039999624093374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,127,0.008922666932145754
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,7,0.008922666932145754
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,255,0.009008000294367472
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,511,0.011098666737476984
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,511,0.010608000059922537
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,1023,0.010911999891201654
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,1023,0.010869332899649939
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,1,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,2047,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,2047,0.011087999989589056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,127,0.008896000062425932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,4095,0.013167999684810638
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,8191,0.015562667200962702
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,8191,0.015077333897352219
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,32767,0.03348266581694285
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,16383,0.03152533372243246
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,15,0.009253333633144697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,32767,0.033439998825391136
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,65535,0.08678932984670003
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,65535,0.05193066596984863
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,1,0.009365333244204521
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,1,0.009301333377758661
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,131071,0.09040533502896626
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,4095,0.013365333278973898
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,1,0.009173333023985228
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,3,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,255,0.009103999783595404
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,7,0.009365333244204521
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,15,0.008863999818762144
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,15,0.009914666414260864
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,31,0.0106133334338665
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,63,0.009173333023985228
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,63,0.00915733352303505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,127,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,131071,0.16085867087046304
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,255,0.009045333291093508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,255,0.009279999881982803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,3,0.008885333314538002
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,511,0.010837333897749582
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,7,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,1023,0.010965333630641302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,2047,0.01109333336353302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,16383,0.01720533271630605
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,4095,0.013237333546082178
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,4095,0.013104000439246496
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,127,0.009008000294367472
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,8191,0.015114666273196539
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,16383,0.047914668917655945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,16383,0.031514666974544525
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,32767,0.08468799789746602
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,32767,0.050197333097457886
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,511,0.010832000523805618
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,65535,0.15657599767049155
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,31,0.009450666606426239
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,32767,0.05060799916585287
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,1,0.009594666461149851
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,131071,0.16211733222007751
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,8191,0.029370665550231934
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,3,0.009279999881982803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,3,0.009162666896979014
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,7,0.009759999811649323
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,15,0.008912000184257826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,15,0.010026666646202406
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,1023,0.01163200040658315
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,65535,0.08820799986521403
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,63,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,63,0.009077333534757296
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,127,0.009317333499590555
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,127,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,255,0.009248000259200731
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,255,0.009050666665037474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,511,0.010992000500361124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,7,0.009312000125646591
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,1023,0.011034666250149408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,1023,0.010928000013033548
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,31,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,31,0.009546666716535887
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,4095,0.02940266579389572
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,4095,0.015344000111023584
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,1,0.009343999748428663
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,8191,0.031248000760873158
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,16383,0.04993066688378652
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,511,0.011071999867757162
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,32767,0.15706666310628256
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,32767,0.08475200335184734
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,2047,0.013077333569526672
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,65535,0.3009546597798665
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,131071,0.30030399560928345
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,65535,0.15702399611473083
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,8191,0.04773866633574168
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,131071,0.5883626540501913
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,16383,0.08396266897519429
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,2047,0.012991999586423239
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,2047,0.011183999478816986
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,131071,0.3027519981066386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,1,0.019173332800467808
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,1,0.01720533271630605
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,3,0.01932266727089882
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,3,0.017194667210181553
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,7,0.019274666905403137
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,7,0.01710933322707812
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,15,0.019120000302791595
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,31,0.01911466692884763
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,15,0.01735466718673706
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,31,0.017301333447297413
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,63,0.019199999670187633
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,63,0.01725333308180173
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,127,0.019253333409627277
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,127,0.01741333305835724
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,255,0.03125333289305369
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,255,0.017621333400408428
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,511,0.032069332897663116
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,511,0.049914668003718056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,1,0.03154666721820831
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,1,0.02754133443037669
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,3,0.03143466760714849
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,3,0.02733866622050603
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,31,0.03145066648721695
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,7,0.02752000093460083
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,31,0.027306665976842243
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,15,0.027600000301996868
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,63,0.03148799886306127
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,7,0.031343999008337654
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,63,0.027461332579453785
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,15,0.0315733328461647
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,127,0.0356480007370313
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,255,0.05080533524354299
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,255,0.035930665830771126
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,511,0.08877333005269368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,511,0.054010664423306785
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,1,0.0484746644894282
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,1,0.05810666580994924
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,127,0.027776000400384266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,3,0.058101331194241844
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,3,0.048197334011395775
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,7,0.0581279993057251
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,15,0.04779199759165446
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,31,0.0581279993057251
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,7,0.04825599988301595
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,31,0.04890666902065277
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,63,0.05818133552869161
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,63,0.048250665267308555
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,15,0.05794133245944977
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,127,0.060517330964406334
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,127,0.05258666475613912
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,255,0.09257599711418152
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,255,0.061349332332611084
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,1,0.0977280040582021
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,3,0.09778133034706116
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,511,0.09622933467229207
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,3,0.08479467034339905
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,1,0.08469333251317342
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,7,0.09708799918492635
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,511,0.1648319959640503
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,7,0.08474133412043254
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,15,0.09814932942390442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,15,0.08403733372688293
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,31,0.08502399921417236
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,63,0.09895466764767964
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,63,0.08471999565760295
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,127,0.08525866270065308
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,31,0.09815999865531921
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,127,0.10127466917037964
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,255,0.16506133476893106
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,255,0.10116799672444661
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,1,0.031370667119820915
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,511,0.3076000014940898
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,1,0.02739199995994568
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,3,0.0315786674618721
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,511,0.16723199685414633
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,3,0.02775999903678894
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,7,0.03162133445342382
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,7,0.028143999477227528
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,15,0.031541332602500916
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,31,0.03140799949566523
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,31,0.02739733209212621
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,15,0.027776000400384266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,63,0.02759466568628947
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,127,0.03591466695070267
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,63,0.0314026673634847
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,127,0.027690666417280834
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,255,0.051967998345692955
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,1,0.05600533386071523
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,255,0.03555733213822047
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,1,0.046949331959088646
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,3,0.05630399783452352
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,7,0.055957332253456116
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,3,0.04644800225893656
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,7,0.04654400050640106
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,15,0.056090667843818665
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,15,0.045994664231936135
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,31,0.0562666654586792
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,31,0.0469813346862793
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,63,0.046442667643229164
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,63,0.05630933245023092
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,127,0.059818665186564125
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,127,0.049882665276527405
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,1,0.10552533467610677
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,255,0.09075733025868733
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,255,0.05929600199063619
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,1,0.09293867150942485
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,3,0.10480533043543498
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,3,0.093231995900472
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,7,0.1051680048306783
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,7,0.09299199779828389
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,15,0.10541333754857381
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,31,0.09294399619102478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,15,0.09282666444778442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,31,0.10520000259081523
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,63,0.10537599523862202
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,63,0.09284266829490662
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,127,0.10959466298421223
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,1,0.18368534247080484
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,127,0.09344533085823059
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,255,0.17307732502619425
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,255,0.10986666878064473
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,1,0.15642133355140686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,3,0.1830293337504069
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,3,0.15666133165359497
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,7,0.15634666879971823
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,7,0.1830880045890808
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,15,0.18413333098093668
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,31,0.15665066242218018
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,63,0.1843199928601583
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,63,0.15692800283432007
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,15,0.15703999996185303
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,31,0.18237332503000894
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,3,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,7,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,1,0.00933333362142245
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,3,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,15,0.009114666531483332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,15,0.009072000160813332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,31,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,31,0.00914666677514712
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,63,0.009162666896979014
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,63,0.00922133338948091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,127,0.009072000160813332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,127,0.010426666587591171
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,255,0.00891733355820179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,255,0.009178666397929192
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,511,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,7,0.00915733352303505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,127,0.19054933389027914
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,1,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,2047,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,4095,0.013093333691358566
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,4095,0.013002666334311167
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,2047,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,8191,0.015061333775520325
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,511,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,16383,0.03136533250411352
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,16383,0.01710933322707812
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,32767,0.05014933149019877
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,65535,0.08746666709582011
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,65535,0.05186133086681366
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,1,0.009077333534757296
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,1,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,255,0.3179413278897603
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,127,0.157151997089386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,255,0.19023466110229492
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,32767,0.0337119996547699
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,15,0.008869333192706108
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,15,0.008933333059151968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,31,0.009205333267649015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,8191,0.015813333292802174
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,31,0.008879999940594038
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,63,0.008949333180983862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,3,0.009141333401203156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,127,0.00892800030608972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,127,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,255,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,511,0.011002667248249054
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,7,0.009050666665037474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,511,0.011034666250149408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,1023,0.011125333607196808
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,1023,0.011098666737476984
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,2047,0.01314666618903478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,63,0.008826666822036108
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,2047,0.01110400011142095
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,4095,0.01302933320403099
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,255,0.008922666932145754
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,8191,0.029877332349618275
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,8191,0.015717333803574245
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,16383,0.04841599861780802
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,16383,0.030623999734719593
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,3,0.009328000247478485
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,32767,0.08667733271916707
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,32767,0.049546668926874794
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,65535,0.15876266360282898
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,65535,0.08768000205357869
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,3,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,3,0.009178666397929192
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,7,0.009338666374484697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,1023,0.011168000598748526
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,15,0.009392000113924345
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,15,0.00996800015370051
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,31,0.009141333401203156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,31,0.009141333401203156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,63,0.009941333283980688
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,63,0.009152000149091085
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,127,0.010058666889866194
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,1,0.009039999917149544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,255,0.009290666629870733
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,7,0.00898133342464765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,255,0.009381333366036415
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,4095,0.015173333386580149
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,511,0.01091733326514562
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,1023,0.011749333391586939
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,1023,0.011002667248249054
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,2047,0.01320533330241839
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,7,0.009322666873534521
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,2047,0.01303999995191892
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,1,0.008922666932145754
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,4095,0.015290666371583939
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,8191,0.03129599988460541
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,16383,0.08433600266774495
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,1023,0.010826667149861654
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,511,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,32767,0.08675733208656311
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,65535,0.3009333411852519
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,4095,0.02977599948644638
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,127,0.009253333633144697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,1,0.009285333255926767
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,3,0.009130666653315226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,3,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,7,0.009141333401203156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,16383,0.049770668148994446
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,15,0.009989333028594652
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,32767,0.15667200088500977
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,31,0.009248000259200731
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,31,0.010037333394090334
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,63,0.010981333752473196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,127,0.011007999380429586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,127,0.010714666297038397
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,8191,0.048021331429481506
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,255,0.011221333096424738
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,255,0.010597333312034607
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,7,0.009610666582981745
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,511,0.011114666859308878
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,1023,0.012069333344697952
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,15,0.00922133338948091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,2047,0.029306667546431225
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,2047,0.015072000523408255
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,4095,0.04771199822425842
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,4095,0.03070933371782303
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,8191,0.04825599988301595
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,8191,0.0830080012480418
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,16383,0.15527466932932535
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,1,0.009248000259200731
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,16383,0.08423999945322673
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,32767,0.29971200227737427
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,1023,0.013104000439246496
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,1,0.009328000247478485
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,65535,0.5884213447570801
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,65535,0.3007040023803711
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,63,0.009312000125646591
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,65535,0.1606933375199636
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,7,0.009103999783595404
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,15,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,511,0.011152000476916632
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,31,0.009056000038981438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,63,0.009183999771873156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,1,0.009008000294367472
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,63,0.009290666629870733
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,3,0.009125333279371262
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,127,0.009290666629870733
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,255,0.009279999881982803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,3,0.008922666932145754
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,7,0.009189333145817121
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,255,0.009183999771873156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,15,0.008938666433095932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,511,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,2047,0.012335999558369318
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,31,0.009050666665037474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,127,0.009258666386206945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,8191,0.029653333127498627
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,8191,0.015173333386580149
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,511,0.010842667271693548
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,1023,0.010965333630641302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,16383,0.03173866619666418
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,2047,0.01110400011142095
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,32767,0.08639466762542725
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,32767,0.049733335773150124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,4095,0.013023999830087027
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,65535,0.16102932890256247
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,1,0.010010666524370512
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,3,0.009429333110650381
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,3,0.009093333035707474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,1023,0.011253333340088526
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,7,0.009984000275532404
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,15,0.009301333377758661
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,1,0.009253333633144697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,31,0.009679999823371569
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,31,0.00922133338948091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,63,0.009226666763424873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,16383,0.04806933303674062
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,127,0.009269333134094873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,7,0.009098666409651438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,127,0.008922666932145754
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,255,0.009242666885256767
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,255,0.00892800030608972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,511,0.011007999380429586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,511,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,1023,0.011338666081428528
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,1023,0.011333333949247995
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,65535,0.08905067046483357
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,2047,0.013290667285521826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,4095,0.015381333728631338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,8191,0.048394665122032166
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,4095,0.015029333531856537
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,8191,0.031354665756225586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,16383,0.08682133754094441
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,16383,0.04983466863632202
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,63,0.009941333283980688
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,32767,0.1593653361002604
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,32767,0.08691733082135518
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,1,0.009583999713261923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,65535,0.30583999554316205
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,65535,0.1613920032978058
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,32767,0.15763200322786966
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,3,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,7,0.010890666395425797
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,7,0.00921066664159298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,15,0.010890666395425797
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,2047,0.01322666679819425
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,31,0.01202133297920227
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,4095,0.029978667696317036
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,63,0.009050666665037474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,63,0.009290666629870733
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,127,0.010879999647537867
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,127,0.009493333597977957
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,255,0.010602666685978571
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,1,0.009290666629870733
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,15,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,3,0.009205333267649015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,1023,0.013194666554530462
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,1023,0.013061333447694778
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,2047,0.029391999046007793
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,2047,0.015285332997639975
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,4095,0.04747200012207031
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,31,0.009103999783595404
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,8191,0.0846506655216217
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,8191,0.04854933420817057
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,16383,0.155349334081014
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,255,0.00898133342464765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,511,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,32767,0.30050132671991986
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,32767,0.15983466307322183
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,1,0.009237333511312803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,1,0.009237333511312803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,3,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,7,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,7,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,16383,0.0867199997107188
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,15,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,15,0.00915733352303505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,31,0.008933333059151968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,31,0.009237333511312803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,63,0.009162666896979014
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,15,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,63,0.009359999870260557
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,127,0.008933333059151968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,255,0.008938666433095932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,65535,0.3052106698354085
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,511,0.01099733387430509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,511,0.01097600037852923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,1023,0.010949333508809408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,2047,0.010842667271693548
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,511,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,2047,0.01099733387430509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,4095,0.010847999403874079
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,4095,0.01099733387430509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,3,0.009029333169261614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,8191,0.01099733387430509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,16383,0.015253332753976187
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,32767,0.019285333653291065
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,1023,0.010938666760921478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,65535,0.02752000093460083
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,65535,0.025407999753952026
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,131071,0.03142933299144109
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,131071,0.02938133229811986
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,65535,0.5881386597951254
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,4095,0.03035199890534083
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,1,0.009136000027259191
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,3,0.008874666566650072
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,3,0.009050666665037474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,32767,0.019248000035683315
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,7,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,15,0.009103999783595404
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,15,0.008933333059151968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,31,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,63,0.009130666653315226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,1,0.009050666665037474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,63,0.009077333534757296
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,16383,0.015119999647140503
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,255,0.00898133342464765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,255,0.008874666566650072
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,7,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,31,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,1023,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,1023,0.011066666493813196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,127,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,2047,0.011034666250149408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,127,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,4095,0.011034666250149408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,4095,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,511,0.011498666057984034
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,8191,0.011002667248249054
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,8191,0.010992000500361124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,16383,0.01504533365368843
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,32767,0.019445333629846573
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,8191,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,32767,0.019141333798567455
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,65535,0.023573334018389385
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,131071,0.04134399940570196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,131071,0.024693332612514496
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,1,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,16383,0.015130666395028433
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,1,0.00922133338948091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,3,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,3,0.009109333157539368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,255,0.009232000137368837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,15,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,15,0.00922133338948091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,31,0.009136000027259191
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,63,0.008938666433095932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,63,0.009045333291093508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,511,0.010911999891201654
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,127,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,7,0.009178666397929192
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,255,0.00915733352303505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,127,0.00956266683836778
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,511,0.010933333386977514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,1023,0.010890666395425797
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,1023,0.010869332899649939
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,127,0.00890666681031386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,2047,0.01097600037852923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,7,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,4095,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,2047,0.011125333607196808
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,8191,0.011178666104873022
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,31,0.009258666386206945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,8191,0.011007999380429586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,16383,0.01587733378012975
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,32767,0.01913600042462349
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,32767,0.01911466692884763
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,65535,0.03584533433119456
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,2047,0.010890666395425797
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,131071,0.05472533404827118
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,131071,0.037248000502586365
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,255,0.008922666932145754
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,1,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,1,0.00919999989370505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,3,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,511,0.009088000282645226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,15,0.00891733355820179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,15,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,31,0.009050666665037474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,31,0.009039999917149544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,63,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,4095,0.011034666250149408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,127,0.00890666681031386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,65535,0.02250133454799652
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,7,0.009269333134094873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,127,0.009077333534757296
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,255,0.008938666433095932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,255,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,511,0.010885333021481832
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,1023,0.011130666981140772
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,2047,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,2047,0.01097600037852923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,63,0.00914666677514712
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,4095,0.011071999867757162
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,4095,0.010672000547250112
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,3,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,8191,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,8191,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,16383,0.015893333901961643
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,32767,0.019205333044131596
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,511,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,32767,0.019733333339293797
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,65535,0.021530665457248688
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,131071,0.042080000042915344
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,1,0.009418666362762451
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,1,0.009375999992092451
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,3,0.009242666885256767
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,7,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,3,0.009338666374484697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,7,0.00922133338948091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,16383,0.015194666882356008
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,15,0.009258666386206945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,15,0.008949333180983862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,65535,0.023728000621000927
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,1023,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,31,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,131071,0.023749334116776783
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,127,0.009183999771873156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,255,0.009077333534757296
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,255,0.009039999917149544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,511,0.01089599976936976
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,16383,0.015130666395028433
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,7,0.009408000235756239
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,1023,0.01126933346192042
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,31,0.008778666456540426
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,1023,0.010890666395425797
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,2047,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,63,0.008933333059151968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,4095,0.011066666493813196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,4095,0.010965333630641302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,8191,0.011077333241701126
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,8191,0.011050666371981302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,16383,0.01525866612792015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,32767,0.019365333020687103
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,511,0.010992000500361124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,32767,0.018474667022625606
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,65535,0.03632533301909765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,65535,0.019167999426523846
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,131071,0.055311997731526695
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,2047,0.0103946669648091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,65535,0.019850666324297588
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,1,0.008912000184257826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,3,0.009237333511312803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,3,0.00903466654320558
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,7,0.00927466650803884
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,15,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,16383,0.01532799998919169
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,15,0.009392000113924345
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,31,0.008938666433095932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,63,0.008938666433095932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,127,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,127,0.00898133342464765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,127,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,1,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,7,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,255,0.008922666932145754
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,511,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,511,0.011007999380429586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,1023,0.011050666371981302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,1023,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,31,0.008901333436369896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,4095,0.011002667248249054
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,4095,0.010960000256697336
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,8191,0.01339200014869372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,63,0.009050666665037474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,255,0.009050666665037474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,32767,0.033615998923778534
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,32767,0.01905599981546402
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,2047,0.010965333630641302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,2047,0.010981333752473196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,65535,0.03542399903138479
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,131071,0.08710933725039165
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,131071,0.053904001911481224
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,131071,0.037605332831541695
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,1,0.009082666908701261
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,16383,0.017029333859682083
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,3,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,7,0.009109333157539368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,7,0.0099093330403169
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,63,0.009205333267649015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,15,0.009626666704813639
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,31,0.009130666653315226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,16383,0.017077332983414333
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,63,0.00926399976015091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,63,0.009253333633144697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,3,0.00916800027092298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,127,0.009285333255926767
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,255,0.009893333539366722
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,255,0.009242666885256767
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,65535,0.0518453319867452
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,511,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,511,0.01121066634853681
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,31,0.009610666582981745
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,2047,0.013866666704416275
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,1,0.009248000259200731
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,2047,0.012954667210578918
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,4095,0.030271999537944794
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,4095,0.015504000087579092
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,8191,0.04901333153247833
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,127,0.009258666386206945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,8191,0.03148266673088074
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,16383,0.05006400247414907
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,15,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,8191,0.013295999417702356
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,32767,0.1609280010064443
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,3,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,3,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,7,0.010922666639089584
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,7,0.009973333527644476
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,15,0.010832000523805618
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,15,0.00892800030608972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,16383,0.08684266606966655
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,31,0.009194666519761086
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,1023,0.011120000233252844
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,1,0.009354666496316591
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,63,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,127,0.010666667173306147
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,127,0.01062400018175443
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,255,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,255,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,511,0.011317333827416102
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,511,0.010938666760921478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,1023,0.01313599944114685
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,1023,0.012805332740147909
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,31,0.010922666639089584
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,2047,0.029450667401154835
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,2047,0.015157333264748255
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,1023,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,4095,0.04834666848182678
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,4095,0.030767999589443207
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,8191,0.08473066488901775
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,8191,0.0498879998922348
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,16383,0.15995200475056967
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,1,0.011781333635250727
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,32767,0.16216533382733664
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,32767,0.08906666437784831
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,1,0.011333333949247995
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,3,0.011264000087976456
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,7,0.011519999553759893
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,7,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,15,0.011711999773979187
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,31,0.011071999867757162
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,1,0.01073066641887029
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,31,0.011359999577204386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,63,0.010965333630641302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,127,0.010949333508809408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,16383,0.08610666791598003
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,255,0.010954666882753372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,32767,0.304693341255188
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,511,0.01309866706530253
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,63,0.009248000259200731
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,1023,0.02942399928967158
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,3,0.011039999624093374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,1023,0.015034666905800501
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,2047,0.04785066843032837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,127,0.010879999647537867
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,4095,0.084714670976003
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,255,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,8191,0.15710399548212686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,8191,0.08742933471997578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,63,0.01108266661564509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,16383,0.301749328772227
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,32767,0.5962773164113363
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,15,0.01110400011142095
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,32767,0.3113706707954407
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,3,0.008869333192706108
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,1,0.008933333059151968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,7,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,3,0.009189333145817121
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,15,0.008853333070874214
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,15,0.008912000184257826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,31,0.00892800030608972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,511,0.013173333058754602
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,63,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,63,0.009077333534757296
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,127,0.00891733355820179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,127,0.00892800030608972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,1,0.00892800030608972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,2047,0.031471999982992806
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,255,0.009045333291093508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,4095,0.04986133178075155
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,1023,0.010922666639089584
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,1023,0.011178666104873022
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,7,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,2047,0.010869332899649939
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,4095,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,4095,0.010890666395425797
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,31,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,16383,0.1625173290570577
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,16383,0.016117333124081295
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,32767,0.01922133316596349
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,16383,0.01504533365368843
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,32767,0.017466666797796886
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,65535,0.03691199918588003
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,511,0.011114666859308878
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,131071,0.05572799841562907
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,131071,0.03765333443880081
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,2047,0.011077333241701126
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,1,0.008863999818762144
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,3,0.009066666786869368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,8191,0.01118933285276095
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,7,0.009045333291093508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,8191,0.011317333827416102
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,7,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,15,0.00903466654320558
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,15,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,31,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,63,0.00919999989370505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,63,0.010496000448862711
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,65535,0.019466667125622433
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,127,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,255,0.00898133342464765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,1,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,3,0.009248000259200731
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,511,0.010197333370645842
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,1023,0.010890666395425797
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,1023,0.011231999844312668
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,2047,0.011898666620254517
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,2047,0.011098666737476984
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,511,0.01099733387430509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,127,0.00898133342464765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,4095,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,255,0.009056000038981438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,16383,0.01752000053723653
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,511,0.010922666639089584
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,16383,0.015834666788578033
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,32767,0.033439998825391136
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,32767,0.019088000059127808
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,65535,0.05193066596984863
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,65535,0.03522666543722153
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,31,0.009082666908701261
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,131071,0.054287999868392944
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,4095,0.011066666493813196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,1,0.009269333134094873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,3,0.009205333267649015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,8191,0.013295999417702356
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,3,0.009183999771873156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,7,0.009056000038981438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,15,0.009317333499590555
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,15,0.009285333255926767
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,31,0.009349333122372627
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,31,0.012576000144084295
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,63,0.00926399976015091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,63,0.00926399976015091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,127,0.00926399976015091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,127,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,1,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,255,0.009808000177145004
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,511,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,8191,0.013141332815090815
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,1023,0.011285333583752314
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,1023,0.010949333508809408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,2047,0.011071999867757162
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,4095,0.013194666554530462
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,8191,0.016927999754746754
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,8191,0.01563199982047081
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,131071,0.0900320013364156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,16383,0.03158933420976003
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,255,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,32767,0.0503359983364741
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,511,0.011050666371981302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,32767,0.03336533407370249
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,7,0.008874666566650072
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,2047,0.011247999966144562
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,4095,0.012938667088747025
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,131071,0.158351997534434
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,1,0.010319999729593595
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,1,0.009514666472872099
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,3,0.01020800011853377
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,255,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,3,0.01032533310353756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,16383,0.017114666601022083
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,7,0.010319999729593595
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,15,0.00997866690158844
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,31,0.010618666807810465
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,31,0.009466666728258133
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,63,0.009973333527644476
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,127,0.010384000216921171
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,127,0.010960000256697336
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,7,0.01097600037852923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,255,0.010053333515922228
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,255,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,15,0.010981333752473196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,65535,0.05220800141493479
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,511,0.01108266661564509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,131071,0.09009066224098206
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,63,0.009984000275532404
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,1023,0.013093333691358566
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,2047,0.029637334247430164
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,4095,0.04837333162625631
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,4095,0.03108799954255422
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,8191,0.08643733461697896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,8191,0.05004799862702688
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,65535,0.08496532837549846
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,511,0.010911999891201654
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,1023,0.013482666263977686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,16383,0.15969066818555197
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,3,0.011823999385039011
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,2047,0.015295999745527903
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,3,0.011087999989589056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,7,0.0122079998254776
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,7,0.011514666179815928
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,15,0.011551999797423681
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,31,0.011749333391586939
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,63,0.011450666934251785
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,1,0.011472000430027643
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,16383,0.0872213343779246
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,1,0.010928000013033548
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,127,0.011312000453472137
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,127,0.011498666057984034
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,255,0.011168000598748526
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,511,0.013088000317414602
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,31,0.011440000186363855
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,1023,0.015344000111023584
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,1023,0.029279999434947968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,63,0.01163200040658315
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,2047,0.05007466673851013
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,4095,0.049813335140546165
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,4095,0.08617599805196126
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,511,0.013760000467300415
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,8191,0.16039466857910156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,15,0.011461333682139715
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,16383,0.30779733260472614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,16383,0.1658560037612915
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,8191,0.08796800176302592
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,1,0.013306666165590286
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,3,0.015002666662136713
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,2047,0.03134933362404505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,7,0.01431999976436297
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,255,0.010992000500361124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,15,0.015157333264748255
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,31,0.014688000082969666
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,31,0.013082666943470636
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,63,0.015040000279744467
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,1,0.013994666437307993
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,63,0.013232000172138214
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,127,0.013077333569526672
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,255,0.015125333021084467
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,3,0.01313599944114685
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,7,0.013280000537633896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,511,0.02956799914439519
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,511,0.01701333373785019
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,1023,0.04881600042184194
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,127,0.015082667271296183
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,2047,0.050586665670077004
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,4095,0.08752533793449402
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,4095,0.15828266739845276
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,8191,0.304202675819397
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,15,0.013327999661366144
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,1023,0.03149333347876867
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,1,0.011317333827416102
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,1,0.011359999577204386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,255,0.01357866699496905
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,8191,0.16184000174204508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,3,0.01129066695769628
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,3,0.012213333199421564
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,2047,0.0846506655216217
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,15,0.011231999844312668
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,15,0.010922666639089584
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,16383,0.30899200836817425
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,31,0.011258666714032492
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,63,0.010981333752473196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,63,0.011247999966144562
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,7,0.011349332829316458
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,127,0.011685332904259363
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,255,0.010960000256697336
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,255,0.011136000355084738
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,16383,0.5972373485565186
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,511,0.015103999525308609
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,511,0.013151999562978745
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,7,0.011754666765530905
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,1023,0.029445332785447437
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,127,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,2047,0.031744000812371574
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,31,0.013034666577974955
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,4095,0.0862666666507721
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,2047,0.049466664592425026
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,4095,0.0505973349014918
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,3,0.014373333503802618
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,3,0.013424000392357508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,1023,0.015135999768972397
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,7,0.013338666409254074
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,7,0.015114666273196539
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,31,0.01524266724785169
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,15,0.01333333303531011
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,31,0.013002666334311167
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,63,0.015103999525308609
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,127,0.014949332922697067
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,127,0.013365333278973898
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,255,0.014314666390419006
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,255,0.012970666090647379
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,511,0.029605334003766377
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,511,0.017157333592573803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,1,0.014373333503802618
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,1023,0.04779199759165446
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,1023,0.03136000037193298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,63,0.013349333157142004
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,4095,0.16032000382741293
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,2047,0.08617066343625386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,2047,0.0499839981396993
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,1,0.01913600042462349
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,4095,0.088128000497818
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,1,0.013408000270525614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,3,0.01964266722400983
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,7,0.017077332983414333
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,15,0.019199999670187633
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,3,0.01722666621208191
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,15,0.017301333447297413
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,31,0.01922133316596349
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,31,0.01714666684468587
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,1,0.017173333714405697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,63,0.01921066641807556
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,63,0.01721599946419398
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,15,0.015040000279744467
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,127,0.017152000218629837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,255,0.029706666866938274
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,255,0.017237332959969837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,511,0.04780266682306925
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,7,0.019744000087181728
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,511,0.03156266609827677
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,2047,0.15596266587575278
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,127,0.019189332922299702
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,4095,0.1627893348534902
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,4095,0.2993013262748718
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,1,0.015216000378131866
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,3,0.015098666151364645
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,2047,0.08681600292523702
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,3,0.013130666067202887
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,7,0.014208000153303146
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,7,0.01302933320403099
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,1,0.01309866706530253
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,15,0.014469332993030548
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,1023,0.08454933762550354
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,15,0.01369599997997284
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,31,0.013173333058754602
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,63,0.013056000073750814
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,127,0.014901333798964819
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,1023,0.05093866586685181
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,255,0.012970666090647379
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,63,0.014709333578745524
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,511,0.030165334542592365
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,127,0.01357866699496905
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,511,0.017125333348910015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,1023,0.04949333270390829
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,1023,0.03136000037193298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,31,0.014896000425020853
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,1,0.01912533367673556
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,1,0.017418666432301205
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,2047,0.05009600023428599
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,3,0.01913600042462349
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,7,0.019280000279347103
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,7,0.017349333812793095
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,15,0.0194560003777345
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,2047,0.08702400326728821
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,31,0.019359999646743137
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,31,0.017157333592573803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,63,0.019088000059127808
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,3,0.01714133347074191
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,63,0.017525333911180496
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,15,0.017173333714405697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,127,0.017152000218629837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,255,0.017055999487638474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,511,0.04877333343029022
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,511,0.03178666780392329
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,255,0.014997333288192749
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,127,0.01921066641807556
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,1023,0.08554133772850037
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,1023,0.05193066596984863
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,1,0.031727999448776245
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,2047,0.08948266506195068
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,3,0.03141866624355316
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,2047,0.15953600406646729
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,255,0.030832000076770782
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,1,0.028597332537174225
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,7,0.027642667293548584
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,7,0.03145600110292435
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,15,0.02769600103298823
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,31,0.027669332921504974
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,63,0.027562665442625683
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,3,0.02754666656255722
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,127,0.02792533238728841
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,63,0.0317493329445521
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,255,0.051311999559402466
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,15,0.031397332747777305
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,255,0.03461333364248276
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,127,0.03544000039498011
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,31,0.03147733211517334
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,1,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,511,0.052943999568621315
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,3,0.008837333569924036
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,7,0.008933333059151968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,7,0.008821333448092142
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,3,0.009061333412925402
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,15,0.008933333059151968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,2047,0.16448533535003662
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,1023,0.15915733575820923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,31,0.00916800027092298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,1023,0.09052266677220662
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,31,0.009279999881982803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,1,0.009072000160813332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,127,0.00921066664159298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,255,0.00891733355820179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,511,0.010933333386977514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,511,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,15,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,1023,0.011237333218256632
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,1023,0.010757333288590113
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,2047,0.010591999938090643
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,127,0.009045333291093508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,4095,0.010911999891201654
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,511,0.08709866801897685
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,8191,0.01357866699496905
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,16383,0.01738133281469345
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,255,0.009290666629870733
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,16383,0.015279999623696009
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,32767,0.0332640012105306
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,32767,0.019215999792019527
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,65535,0.053114667534828186
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,2047,0.3018239935239156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,65535,0.03552533437808355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,131071,0.08981333176294963
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,63,0.008949333180983862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,1,0.009082666908701261
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,3,0.008879999940594038
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,63,0.008885333314538002
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,3,0.008890666688481966
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,7,0.009173333023985228
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,7,0.008879999940594038
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,15,0.008821333448092142
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,8191,0.013130666067202887
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,31,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,31,0.009290666629870733
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,63,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,127,0.009306666751702627
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,131071,0.0539626677831014
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,127,0.009205333267649015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,255,0.009408000235756239
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,511,0.011034666250149408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,511,0.01091733326514562
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,15,0.008885333314538002
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,1023,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,1023,0.011050666371981302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,2047,0.010901333143313726
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,2047,0.011034666250149408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,4095,0.013045333325862885
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,4095,0.0129120002190272
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,2047,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,8191,0.015119999647140503
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,4095,0.011146667102972666
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,32767,0.0505973349014918
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,32767,0.03346666693687439
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,65535,0.08703466256459554
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,65535,0.05296533306439718
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,131071,0.16082132856051126
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,131071,0.09089066584904988
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,1,0.009125333279371262
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,1,0.009317333499590555
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,3,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,3,0.00919999989370505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,7,0.009039999917149544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,7,0.009237333511312803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,15,0.00927466650803884
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,31,0.009136000027259191
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,63,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,31,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,63,0.00903466654320558
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,63,0.009189333145817121
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,8191,0.015439999600251516
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,16383,0.03264000018437704
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,255,0.009237333511312803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,255,0.00915733352303505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,511,0.010858666151762009
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,15,0.009061333412925402
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,1023,0.011034666250149408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,1023,0.011050666371981302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,2047,0.012949333836634954
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,1,0.009178666397929192
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,4095,0.012928000340859095
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,127,0.00921066664159298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,16383,0.017269333203633625
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,8191,0.01543466622630755
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,16383,0.04770666857560476
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,511,0.010954666882753372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,16383,0.03140799949566523
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,32767,0.08467732866605122
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,2047,0.01108266661564509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,4095,0.013381333400805792
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,131071,0.30214399099349976
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,127,0.00903466654320558
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,255,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,32767,0.0498933345079422
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,65535,0.15702399611473083
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,65535,0.08912533521652222
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,131071,0.16075733304023743
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,1,0.019296000401178997
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,1,0.017114666601022083
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,3,0.019258666783571243
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,8191,0.029301332930723827
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,7,0.019589333484570186
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,7,0.017711999515692394
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,15,0.019130667050679524
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,15,0.017173333714405697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,31,0.01923199991385142
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,31,0.017162666966517765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,3,0.017477333545684814
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,63,0.019215999792019527
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,63,0.01749333366751671
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,127,0.017082666357358296
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,255,0.030063999195893604
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,255,0.017029333859682083
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,511,0.048911998669306435
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,511,0.0317493329445521
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,1023,0.08673600355784099
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,1,0.03148266673088074
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,1023,0.051088000337282814
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,127,0.0194560003777345
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,1,0.027493332823117573
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,3,0.03147733211517334
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,3,0.027402666707833607
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,7,0.03162666658560435
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,15,0.031557333966096245
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,7,0.02740799884001414
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,31,0.031712000568707786
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,15,0.02779199928045273
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,63,0.03143999973932902
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,63,0.027456000447273254
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,127,0.035589332381884255
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,127,0.027221334477265675
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,31,0.027450665831565857
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,255,0.05153066913286845
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,255,0.03559466699759165
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,511,0.08887466788291931
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,511,0.053301334381103516
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,1,0.05806399881839752
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,1023,0.1604639987150828
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,1,0.048058668772379555
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,1023,0.09011200070381165
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,3,0.05804799993832906
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,7,0.05704533557097117
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,3,0.04833599925041199
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,15,0.05807999769846598
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,7,0.04831466575463613
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,31,0.05808533231417338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,63,0.04795200129350027
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,63,0.05835733314355215
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,15,0.04819199939568838
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,127,0.06053866446018219
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,31,0.048138668139775596
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,127,0.05235200126965841
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,255,0.061424002051353455
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,255,0.09144000212351482
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,511,0.1627893348534902
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,1,0.03143999973932902
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,511,0.09517866373062134
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,1,0.02735999971628189
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,1023,0.16880534092585245
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,1023,0.30709866682688397
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,3,0.03149333347876867
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,3,0.027456000447273254
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,15,0.03152533372243246
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,7,0.03141866624355316
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,15,0.027269333600997925
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,7,0.027610667049884796
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,63,0.031354665756225586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,63,0.02740799884001414
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,127,0.035786665976047516
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,31,0.02741866558790207
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,127,0.02739733209212621
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,255,0.05206400156021118
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,31,0.03145600110292435
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,255,0.03587199995915095
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,511,0.08859733740488689
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,511,0.05398933092753092
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,3,0.05593066910902659
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,1,0.056015998125076294
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,3,0.048010667165120445
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,7,0.056159997979799904
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,7,0.04667733112970988
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,1,0.047744000951449074
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,15,0.04587733248869578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,31,0.056287998954455055
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,31,0.04602666695912679
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,63,0.0562720000743866
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,63,0.04589866598447164
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,15,0.055999999245007835
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,127,0.0601440022389094
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,127,0.05110933383305868
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,255,0.09186133742332458
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,1,0.10514666636784871
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,1,0.09274133046468098
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,511,0.16459733247756958
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,511,0.09335999687512715
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,3,0.09292266766230266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,3,0.10377599795659383
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,255,0.059706668059031166
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,7,0.10352533062299092
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,7,0.09156266848246257
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,15,0.10475200414657593
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,31,0.0928000013033549
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,63,0.10588799913724263
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,15,0.09289600451787312
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,63,0.09226133426030476
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,31,0.10457066694895427
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,1,0.009103999783595404
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,127,0.10929600397745769
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,1,0.009072000160813332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,3,0.008885333314538002
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,7,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,15,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,127,0.0941386620203654
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,15,0.009258666386206945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,7,0.009477333476146063
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,31,0.008874666566650072
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,63,0.009269333134094873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,3,0.009029333169261614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,127,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,255,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,31,0.009189333145817121
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,255,0.17114667097727457
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,511,0.011007999380429586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,1023,0.011157333850860596
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,1023,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,63,0.009093333035707474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,2047,0.01090666651725769
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,127,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,2047,0.010949333508809408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,255,0.10944533348083496
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,4095,0.01309866706530253
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,8191,0.015365333606799444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,255,0.009237333511312803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,16383,0.032138665517171226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,32767,0.0503359983364741
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,8191,0.016549333930015564
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,32767,0.03349333256483078
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,65535,0.0883733332157135
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,511,0.011066666493813196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,65535,0.05197866757710775
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,1,0.00926399976015091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,131071,0.1620213290055593
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,131071,0.09321600198745728
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,3,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,16383,0.017194667210181553
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,7,0.008853333070874214
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,7,0.009253333633144697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,511,0.3148426612218221
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,511,0.17683732509613037
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,4095,0.01309866706530253
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,63,0.00926399976015091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,127,0.00915733352303505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,1,0.009306666751702627
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,127,0.009290666629870733
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,255,0.00914666677514712
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,15,0.008933333059151968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,15,0.00922133338948091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,511,0.010933333386977514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,1023,0.010858666151762009
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,1023,0.01099733387430509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,2047,0.011541333049535751
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,2047,0.011136000355084738
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,4095,0.013541333377361298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,4095,0.013056000073750814
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,8191,0.02935466667016347
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,8191,0.015482666591803232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,16383,0.04798933366934458
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,16383,0.03145600110292435
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,31,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,511,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,32767,0.04981866478919983
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,65535,0.16014400124549866
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,65535,0.08711999654769897
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,131071,0.3054826656977336
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,3,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,1,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,255,0.009072000160813332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,3,0.009194666519761086
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,31,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,7,0.009642666826645533
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,32767,0.08496000369389851
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,15,0.009050666665037474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,15,0.009039999917149544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,31,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,63,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,63,0.008896000062425932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,1,0.009258666386206945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,131071,0.16221333543459573
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,127,0.00996800015370051
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,511,0.010938666760921478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,511,0.010847999403874079
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,63,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,1023,0.011973333855470022
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,1023,0.011007999380429586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,31,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,2047,0.012986666212479273
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,2047,0.013045333325862885
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,4095,0.029535998900731403
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,8191,0.048058668772379555
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,255,0.00903466654320558
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,3,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,16383,0.049072002371152244
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,7,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,32767,0.08669867118199666
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,65535,0.3003893295923869
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,4095,0.015103999525308609
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,127,0.00916800027092298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,8191,0.03148799886306127
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,255,0.00961599995692571
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,1,0.00915733352303505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,3,0.009296000003814697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,3,0.009056000038981438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,32767,0.15625066558519998
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,7,0.009194666519761086
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,7,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,15,0.00891733355820179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,15,0.009301333377758661
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,31,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,63,0.009312000125646591
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,63,0.009178666397929192
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,127,0.009066666786869368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,1,0.008816000074148178
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,255,0.00933333362142245
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,255,0.00927466650803884
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,511,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,511,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,1023,0.011087999989589056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,65535,0.16242133577664694
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,31,0.00884799969693025
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,2047,0.012821332861979803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,2047,0.011658667276302973
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,127,0.009674666449427605
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,4095,0.015125333021084467
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,8191,0.030458666384220123
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,8191,0.015178666760524115
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,16383,0.048954665660858154
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,32767,0.08648533622423808
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,1023,0.011231999844312668
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,32767,0.050810664892196655
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,65535,0.16109866897265115
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,131071,0.3064853350321452
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,131071,0.3084266583124797
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,1,0.009253333633144697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,1,0.009050666665037474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,3,0.009253333633144697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,3,0.009269333134094873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,7,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,4095,0.013050666699806849
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,15,0.009232000137368837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,15,0.009583999713261923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,31,0.008933333059151968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,31,0.00956266683836778
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,63,0.009290666629870733
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,63,0.009232000137368837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,127,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,127,0.009077333534757296
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,255,0.009152000149091085
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,255,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,16383,0.08463467160860698
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,511,0.010832000523805618
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,511,0.010858666151762009
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,1023,0.011920000116030375
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,2047,0.01333333303531011
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,131071,0.16579733292261759
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,4095,0.029722665747006733
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,4095,0.015109332899252573
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,8191,0.048810665806134544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,65535,0.08802666266759236
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,131071,0.589194655418396
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,16383,0.08463467160860698
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,16383,0.05005866785844167
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,32767,0.15843199690183005
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,1023,0.010954666882753372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,32767,0.08681066830952962
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,2047,0.012991999586423239
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,65535,0.30717867612838745
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,1,0.009136000027259191
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,3,0.009338666374484697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,1,0.009279999881982803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,3,0.008837333569924036
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,7,0.009328000247478485
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,131071,0.5989013512929281
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,15,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,31,0.009008000294367472
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,15,0.008922666932145754
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,31,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,63,0.008890666688481966
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,63,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,127,0.008933333059151968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,127,0.009136000027259191
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,255,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,255,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,511,0.009029333169261614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,511,0.010090666512648264
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,1023,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,8191,0.03139200061559677
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,1023,0.011130666981140772
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,2047,0.010949333508809408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,2047,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,4095,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,8191,0.011130666981140772
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,8191,0.010773333410422007
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,16383,0.015087999403476715
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,16383,0.014053333550691605
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,32767,0.019205333044131596
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,65535,0.16179733475049338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,131071,0.30962665875752765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,65535,0.026965332527955372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,7,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,7,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,1,0.009242666885256767
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,1,0.008837333569924036
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,3,0.009279999881982803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,3,0.008869333192706108
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,7,0.009258666386206945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,15,0.009408000235756239
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,65535,0.02737066646416982
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,31,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,31,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,131071,0.030031998952229817
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,63,0.009103999783595404
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,63,0.009029333169261614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,127,0.008949333180983862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,255,0.009365333244204521
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,7,0.008885333314538002
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,255,0.008938666433095932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,511,0.010885333021481832
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,15,0.009194666519761086
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,1023,0.011322667201360067
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,1023,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,131071,0.03146133323510488
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,2047,0.011258666714032492
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,4095,0.01118933285276095
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,127,0.00892800030608972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,8191,0.010949333508809408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,16383,0.031386665999889374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,8191,0.010901333143313726
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,16383,0.015050667027632395
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,16383,0.015114666273196539
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,32767,0.019258666783571243
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,65535,0.023962666591008503
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,65535,0.023333333432674408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,131071,0.042693331837654114
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,131071,0.02420266717672348
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,1,0.00898133342464765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,3,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,4095,0.011050666371981302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,3,0.009466666728258133
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,7,0.008938666433095932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,32767,0.01746133342385292
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,15,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,15,0.00891733355820179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,31,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,31,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,63,0.008933333059151968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,63,0.00890666681031386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,127,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,1,0.00891733355820179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,255,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,4095,0.011114666859308878
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,511,0.009589333087205887
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,1023,0.011087999989589056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,1023,0.013738666971524557
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,32767,0.020138667275508244
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,2047,0.011050666371981302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,2047,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,4095,0.011055999745925268
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,127,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,8191,0.011066666493813196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,8191,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,16383,0.015178666760524115
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,16383,0.01504533365368843
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,511,0.009290666629870733
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,32767,0.019226666539907455
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,32767,0.019226666539907455
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,65535,0.021594665944576263
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,131071,0.04318933188915253
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,131071,0.024058667321999867
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,1,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,1,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,3,0.00884799969693025
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,3,0.009205333267649015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,7,0.009008000294367472
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,2047,0.01089599976936976
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,15,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,15,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,7,0.00927466650803884
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,65535,0.023455999791622162
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,63,0.009119999905427298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,511,0.010960000256697336
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,127,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,4095,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,255,0.009008000294367472
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,255,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,511,0.010330666477481524
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,511,0.01099733387430509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,255,0.00892800030608972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,1023,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,2047,0.010938666760921478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,2047,0.01089599976936976
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,4095,0.010938666760921478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,4095,0.011077333241701126
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,8191,0.010954666882753372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,8191,0.01109333336353302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,16383,0.01534933348496755
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,127,0.009232000137368837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,32767,0.019248000035683315
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,32767,0.018757333358128864
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,7,0.00903466654320558
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,1023,0.010992000500361124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,131071,0.054378668467203774
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,131071,0.03942933430274328
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,63,0.009205333267649015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,1,0.009077333534757296
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,3,0.009306666751702627
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,3,0.009269333134094873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,16383,0.015386667102575302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,7,0.00933333362142245
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,65535,0.03554133325815201
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,7,0.009136000027259191
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,65535,0.0200853335360686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,15,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,31,0.00892800030608972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,63,0.00973866693675518
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,31,0.008816000074148178
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,63,0.009322666873534521
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,1,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,127,0.00922133338948091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,127,0.009519999846816063
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,255,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,511,0.011146667102972666
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,511,0.011274666835864386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,1023,0.010954666882753372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,31,0.008949333180983862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,2047,0.013050666699806849
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,31,0.009125333279371262
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,4095,0.02991466720898946
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,4095,0.015077333897352219
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,8191,0.04839999973773956
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,8191,0.031328000128269196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,16383,0.086709330479304
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,1023,0.011354666203260422
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,32767,0.1607039968172709
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,32767,0.08873599767684937
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,2047,0.013482666263977686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,1,0.011034666250149408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,65535,0.16172800461451212
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,1,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,255,0.00890666681031386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,3,0.010981333752473196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,7,0.009269333134094873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,7,0.009248000259200731
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,15,0.009018666421373686
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,15,0.010661333799362183
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,31,0.009749333063761393
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,63,0.01081066702802976
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,127,0.009685333197315535
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,127,0.009194666519761086
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,65535,0.3073973258336385
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,255,0.009162666896979014
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,3,0.00926399976015091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,511,0.01128000020980835
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,16383,0.0498879998922348
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,1023,0.01310933381319046
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,1023,0.011786667009194693
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,15,0.009296000003814697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,2047,0.014981333166360855
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,4095,0.030031998952229817
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,4095,0.047914668917655945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,255,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,8191,0.04849066833655039
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,16383,0.1602186659971873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,511,0.010960000256697336
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,16383,0.08669333656628926
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,32767,0.30483200152715045
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,31,0.009296000003814697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,1,0.009173333023985228
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,1,0.009114666531483332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,2047,0.030218665798505146
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,65535,0.30851199229558307
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,3,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,3,0.009258666386206945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,8191,0.08572799960772197
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,7,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,15,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,31,0.008938666433095932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,63,0.008949333180983862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,127,0.00891733355820179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,63,0.009413333609700203
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,255,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,255,0.008933333059151968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,511,0.009285333255926767
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,65535,0.5989120006561279
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,511,0.009888000165422758
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,15,0.009322666873534521
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,31,0.009082666908701261
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,2047,0.011071999867757162
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,63,0.008885333314538002
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,2047,0.010992000500361124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,4095,0.010954666882753372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,8191,0.011114666859308878
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,32767,0.16182933251063028
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,16383,0.015578666081031164
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,16383,0.015103999525308609
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,32767,0.019215999792019527
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,32767,0.017845333864291508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,65535,0.03658133248488108
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,7,0.009066666786869368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,131071,0.0547680010398229
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,131071,0.037946666280428566
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,1023,0.010954666882753372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,1,0.009045333291093508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,1,0.009248000259200731
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,3,0.00890666681031386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,7,0.009818666925032934
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,127,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,15,0.008912000184257826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,8191,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,15,0.008912000184257826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,31,0.00921066664159298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,63,0.00892800030608972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,63,0.00914666677514712
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,65535,0.020181333025296528
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,127,0.008890666688481966
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,255,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,1023,0.010869332899649939
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,3,0.009050666665037474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,4095,0.011418666690587997
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,7,0.009061333412925402
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,1023,0.011285333583752314
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,1023,0.011039999624093374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,31,0.009066666786869368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,2047,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,4095,0.010805333654085795
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,4095,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,255,0.009045333291093508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,8191,0.013354666531085968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,16383,0.01718933383623759
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,16383,0.015306666493415833
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,511,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,32767,0.03341866781314214
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,32767,0.01916266605257988
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,511,0.010410666465759277
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,131071,0.08764800429344177
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,127,0.008858666444818178
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,1,0.009989333028594652
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,8191,0.013199999928474426
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,1,0.009973333527644476
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,3,0.009957333405812582
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,3,0.009882666791478792
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,65535,0.05184000233809153
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,7,0.00996800015370051
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,15,0.009599999835093817
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,131071,0.05435733497142792
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,31,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,31,0.009952000031868616
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,63,0.010191999996701876
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,63,0.01110400011142095
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,127,0.009973333527644476
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,127,0.009706666693091393
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,255,0.010922666639089584
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,255,0.00972800018886725
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,511,0.010992000500361124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,511,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,65535,0.03521066655715307
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,1023,0.013088000317414602
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,1023,0.013088000317414602
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,2047,0.011274666835864386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,2047,0.02938133229811986
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,2047,0.015189333508412043
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,8191,0.08477333188056946
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,4095,0.04857600231965383
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,8191,0.05012799799442291
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,4095,0.03154666721820831
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,16383,0.1590559979279836
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,16383,0.08689600229263306
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,1,0.011231999844312668
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,7,0.009818666925032934
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,1,0.010965333630641302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,32767,0.30802667140960693
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,7,0.011434666812419891
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,3,0.011183999478816986
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,7,0.011125333607196808
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,15,0.011328000575304031
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,15,0.011002667248249054
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,31,0.011338666081428528
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,31,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,127,0.011418666690587997
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,127,0.01126933346192042
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,255,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,15,0.010005333150426546
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,255,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,511,0.013183999806642532
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,32767,0.16145599881807962
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,1023,0.029477333029111225
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,63,0.010922666639089584
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,2047,0.04785599807898203
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,4095,0.08562666177749634
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,3,0.011157333850860596
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,8191,0.15985600153605142
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,1023,0.015141333142916361
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,63,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,16383,0.16455466548601785
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,2047,0.031504000226656594
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,1,0.011231999844312668
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,511,0.015168000012636185
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,32767,0.6017813285191854
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,1,0.011274666835864386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,16383,0.3065333366394043
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,3,0.011173332730929056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,3,0.011359999577204386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,4095,0.05021866659323374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,7,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,15,0.011045332998037338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,15,0.011242666592200598
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,31,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,63,0.011130666981140772
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,32767,0.3147839903831482
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,127,0.01128000020980835
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,127,0.011658667276302973
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,255,0.010981333752473196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,7,0.011711999773979187
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,511,0.014277332772811254
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,8191,0.08785600463549297
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,255,0.011061333119869232
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,511,0.012970666090647379
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,1023,0.02978666623433431
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,63,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,1023,0.01578666642308235
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,2047,0.04877866804599762
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,2047,0.03201066702604294
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,8191,0.16013866662979126
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,31,0.011855999628702799
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,4095,0.0862559974193573
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,1,0.015018666783968607
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,1,0.013349333157142004
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,3,0.01414399966597557
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,7,0.013082666943470636
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,7,0.014981333166360855
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,15,0.013306666165590286
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,31,0.015061333775520325
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,8191,0.08957333366076152
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,31,0.013264000415802002
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,63,0.013104000439246496
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,127,0.015034666905800501
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,127,0.013653332988421122
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,15,0.015034666905800501
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,255,0.013050666699806849
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,511,0.030005333324273426
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,511,0.017263999829689663
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,1023,0.04974933465321859
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,1023,0.032101333141326904
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,2047,0.08608532945315044
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,255,0.01451733335852623
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,2047,0.05045866469542185
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,4095,0.05044800043106079
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,63,0.015013333410024643
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,4095,0.08706667025883992
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,3,0.013354666531085968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,8191,0.1620746652285258
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,3,0.015040000279744467
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,8191,0.3089759945869446
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,3,0.013237333546082178
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,1,0.015418666104475657
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,1,0.013605333864688873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,4095,0.16089600324630737
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,7,0.013061333447694778
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,15,0.015146666516860327
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,31,0.015082667271296183
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,15,0.013023999830087027
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,31,0.013274667163689932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,63,0.015077333897352219
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,7,0.015168000012636185
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,127,0.01509333277742068
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,127,0.013199999928474426
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,255,0.014639999717473984
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,255,0.013125333935022354
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,511,0.029658667743206024
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,63,0.013077333569526672
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,1023,0.03259200106064478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,1023,0.04960533479849497
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,2047,0.08678399523099263
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,1,0.01912533367673556
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,511,0.017018667111794155
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,1,0.01740266631046931
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,3,0.019509332875410717
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,4095,0.16142400105794272
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,4095,0.08819733063379924
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,7,0.01913600042462349
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,2047,0.05055999755859375
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,15,0.019141333798567455
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,7,0.017130666722853977
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,3,0.01714666684468587
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,63,0.01915733392039935
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,127,0.019120000302791595
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,15,0.01758933315674464
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,63,0.017162666966517765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,31,0.019333332777023315
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,31,0.01746133342385292
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,127,0.01710933322707812
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,511,0.04841599861780802
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,255,0.029391999046007793
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,511,0.03170666595300039
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,255,0.01716800034046173
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,1023,0.08454400300979614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,1,0.009248000259200731
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,1,0.009162666896979014
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,3,0.009301333377758661
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,7,0.009253333633144697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,15,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,4095,0.304202675819397
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,15,0.009290666629870733
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,31,0.009328000247478485
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,31,0.008922666932145754
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,63,0.00919999989370505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,63,0.009183999771873156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,1023,0.05026133358478546
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,2047,0.08745599786440532
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,7,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,511,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,511,0.010245333115259806
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,4095,0.16218666235605875
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,1023,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,2047,0.15917866428693137
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,2047,0.01108266661564509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,1023,0.011066666493813196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,255,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,127,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,4095,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,8191,0.013381333400805792
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,255,0.008997333546479544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,16383,0.017162666966517765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,3,0.009077333534757296
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,2047,0.010762666662534079
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,32767,0.03412266572316488
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,65535,0.051962668697039284
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,127,0.009189333145817121
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,4095,0.01090666651725769
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,131071,0.05390933156013489
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,8191,0.013424000392357508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,1,0.009359999870260557
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,3,0.008933333059151968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,3,0.009269333134094873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,7,0.009312000125646591
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,16383,0.015082667271296183
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,32767,0.019141333798567455
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,15,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,65535,0.03554133325815201
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,31,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,63,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,63,0.008874666566650072
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,127,0.00922133338948091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,1,0.00916800027092298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,255,0.009205333267649015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,511,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,511,0.010911999891201654
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,1023,0.010965333630641302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,15,0.009130666653315226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,2047,0.01097600037852923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,31,0.009114666531483332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,4095,0.013130666067202887
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,4095,0.0129120002190272
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,127,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,8191,0.015040000279744467
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,16383,0.01738133281469345
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,32767,0.04990399877230326
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,1023,0.010938666760921478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,32767,0.033333333830038704
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,65535,0.08662399649620056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,65535,0.051818668842315674
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,2047,0.010911999891201654
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,131071,0.1609599987665812
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,131071,0.08990400036176045
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,8191,0.0161920003592968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,131071,0.0897813340028127
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,255,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,7,0.00922133338948091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,16383,0.03213333338499069
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,1,0.01786133274435997
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,3,0.01725333308180173
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,7,0.019440000255902607
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,7,0.017152000218629837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,15,0.01956266661485036
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,3,0.019226666539907455
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,15,0.017509333789348602
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,31,0.0194560003777345
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,31,0.017157333592573803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,63,0.017429333180189133
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,1,0.019205333044131596
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,127,0.01960533360640208
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,63,0.01915733392039935
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,255,0.029802667597929638
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,511,0.047872001926104225
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,511,0.031248000760873158
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,1023,0.08639466762542725
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,127,0.017770666629076004
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,255,0.017802666872739792
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,1023,0.05046399931112925
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,2047,0.15875200430552164
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,2047,0.08885866403579712
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,1,0.03148799886306127
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,1,0.02735466758410136
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,3,0.028522667785485584
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,3,0.03153600047032038
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,7,0.031445334355036415
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,7,0.02775999903678894
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,31,0.0327360009153684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,31,0.027669332921504974
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,63,0.03175999969244003
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,63,0.02737066646416982
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,127,0.035536001125971474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,15,0.03145066648721695
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,127,0.027722666660944622
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,255,0.05056533217430115
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,255,0.03558400024970373
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,15,0.02733866622050603
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,511,0.05389333268006643
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,1023,0.16157866517702738
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,511,0.0867039958635966
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,1023,0.09082133571306865
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,1,0.031504000226656594
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,2047,0.30550400416056317
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,1,0.027482666075229645
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,3,0.03153600047032038
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,2047,0.16391467054684958
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,3,0.027701333165168762
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,7,0.03152533372243246
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,7,0.027482666075229645
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,15,0.03142400085926056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,31,0.031514666974544525
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,31,0.027642667293548584
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,15,0.02769600103298823
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,63,0.027600000301996868
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,63,0.031311998764673867
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,127,0.028101332485675812
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,255,0.050944000482559204
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,255,0.03551466763019562
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,127,0.035386666655540466
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,511,0.08830933769543965
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,511,0.0536106675863266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,1023,0.16332800189654031
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,1,0.05587733288606008
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,1023,0.09038933118184407
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,1,0.046240001916885376
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,3,0.056277334690093994
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,3,0.04673600196838379
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,7,0.05599466462930044
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,7,0.04744533201058706
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,15,0.04614933331807455
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,15,0.0561653325955073
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,31,0.04611733555793762
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,63,0.04619200030962626
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,31,0.056143999099731445
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,63,0.05624533196290334
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,127,0.05884799857934316
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,1,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,127,0.05050133168697357
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,1,0.008869333192706108
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,3,0.008922666932145754
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,7,0.008863999818762144
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,255,0.09073066711425781
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,7,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,15,0.009232000137368837
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,31,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,31,0.00927466650803884
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,63,0.00891733355820179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,63,0.009269333134094873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,127,0.008901333436369896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,127,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,511,0.09276800354321797
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,511,0.1641973356405894
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,255,0.009056000038981438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,511,0.010949333508809408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,1023,0.010965333630641302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,1023,0.011125333607196808
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,15,0.00892800030608972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,2047,0.011055999745925268
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,4095,0.012960000584522883
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,255,0.009130666653315226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,511,0.010933333386977514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,4095,0.013167999684810638
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,8191,0.01526933287580808
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,8191,0.015205333630243937
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,16383,0.03212266663710276
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,32767,0.05089599887530009
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,32767,0.033514666060606636
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,65535,0.08798399567604065
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,65535,0.05211733281612396
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,3,0.008976000050703684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,131071,0.16249600052833557
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,131071,0.09179733196894328
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,1,0.00919999989370505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,1,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,3,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,1023,0.3097333312034607
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,7,0.009125333279371262
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,1023,0.16662933429082236
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,15,0.008885333314538002
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,255,0.05845333139101664
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,31,0.008874666566650072
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,63,0.009226666763424873
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,2047,0.010949333508809408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,127,0.009162666896979014
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,127,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,255,0.009056000038981438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,7,0.00897066667675972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,16383,0.017290666699409485
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,511,0.011077333241701126
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,63,0.00892800030608972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,1023,0.01119999960064888
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,1023,0.011231999844312668
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,2047,0.012869333227475485
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,2047,0.01089599976936976
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,4095,0.014767999450365702
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,4095,0.01314666618903478
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,8191,0.015722667177518208
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,16383,0.04805333415667216
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,16383,0.03139200061559677
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,255,0.008933333059151968
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,3,0.009194666519761086
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,32767,0.05019199848175049
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,511,0.01097600037852923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,15,0.009183999771873156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,1,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,131071,0.3063146670659383
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,131071,0.16325866182645163
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,1,0.009162666896979014
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,3,0.00898133342464765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,3,0.009082666908701261
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,7,0.00895999992887179
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,7,0.00914666677514712
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,32767,0.08641067147254944
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,15,0.008874666566650072
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,15,0.009098666409651438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,31,0.008943999807039896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,31,0.008896000062425932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,63,0.009248000259200731
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,63,0.009312000125646591
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,127,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,31,0.009045333291093508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,255,0.00890666681031386
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,255,0.008863999818762144
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,511,0.011055999745925268
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,511,0.010426666587591171
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,1023,0.01126933346192042
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,1023,0.011087999989589056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,2047,0.012661332885424295
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,4095,0.014560000350077948
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,4095,0.01302933320403099
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,65535,0.08737599849700928
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,8191,0.02997333308060964
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,16383,0.04819199939568838
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,127,0.009061333412925402
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,16383,0.03140799949566523
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,32767,0.08642666538556416
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,8191,0.029626667499542236
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,65535,0.15864533185958862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,2047,0.011477333803971609
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,1,0.008869333192706108
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,1,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,131071,0.16191466649373373
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,65535,0.1600160002708435
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,7,0.009056000038981438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,8191,0.015439999600251516
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,15,0.00903466654320558
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,31,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,32767,0.05003733436266581
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,63,0.009045333291093508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,65535,0.08604266246159871
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,63,0.009056000038981438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,127,0.00921066664159298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,127,0.00915733352303505
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,3,0.008879999940594038
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,255,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,511,0.011285333583752314
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,1023,0.01118933285276095
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,1023,0.011136000355084738
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,31,0.009088000282645226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,2047,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,4095,0.011066666493813196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,4095,0.010949333508809408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,3,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,8191,0.011071999867757162
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,8191,0.010944000134865442
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,7,0.008901333436369896
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,16383,0.015173333386580149
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,15,0.009189333145817121
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,32767,0.019071999937295914
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,65535,0.027322667340437572
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,65535,0.02749866743882497
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,131071,0.03147733211517334
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,131071,0.03148799886306127
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,1,0.009114666531483332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,3,0.009130666653315226
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,131071,0.3081653316815694
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,3,0.009045333291093508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,511,0.00960533320903778
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,7,0.009114666531483332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,15,0.008863999818762144
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,15,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,31,0.009061333412925402
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,2047,0.010885333021481832
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,63,0.009066666786869368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,1,0.009002666920423508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,127,0.00903466654320558
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,255,0.008938666433095932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,255,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,255,0.009205333267649015
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,511,0.011231999844312668
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,7,0.00892800030608972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,511,0.01099733387430509
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,32767,0.018186666071414948
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,2047,0.011002667248249054
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,2047,0.010693332801262537
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,4095,0.010778666784365972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,63,0.009237333511312803
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,8191,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,8191,0.011183999478816986
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,16383,0.015285332997639975
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,32767,0.019354666272799175
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,32767,0.019359999646743137
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,1023,0.01129066695769628
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,31,0.008837333569924036
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,65535,0.022858666876951855
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,131071,0.042319998145103455
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,1,0.009583999713261923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,127,0.009056000038981438
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,16383,0.015130666395028433
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,1,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,3,0.009008000294367472
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,3,0.00927466650803884
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,16383,0.015178666760524115
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,7,0.009125333279371262
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,1023,0.010992000500361124
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,15,0.009141333401203156
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,4095,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,31,0.008949333180983862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,63,0.009632000078757605
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,63,0.008949333180983862
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,127,0.009488000224033991
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,127,0.010437333335479101
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,255,0.009850666547815004
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,255,0.009253333633144697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,65535,0.02367466688156128
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,511,0.011007999380429586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,131071,0.02474133421977361
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,511,0.011039999624093374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,1023,0.010970667004585266
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,2047,0.013728000223636627
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,2047,0.012885333349307379
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,4095,0.02941333254178365
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,8191,0.04841066896915436
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,8191,0.03141333411137263
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,16383,0.08673600355784099
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,16383,0.05016533533732096
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,7,0.009429333110650381
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,32767,0.16087466478347778
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,32767,0.08742400010426839
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,1023,0.011253333340088526
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,65535,0.3083466688791911
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,65535,0.16287466883659363
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,4095,0.01526933287580808
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,3,0.009125333279371262
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,131071,0.3120693365732829
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,3,0.009317333499590555
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,7,0.009045333291093508
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,15,0.00921066664159298
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,15,0.009066666786869368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,31,0.009343999748428663
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,15,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,63,0.009312000125646591
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,63,0.009039999917149544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,1,0.00938666673998038
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,127,0.009039999917149544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,255,0.009061333412925402
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,255,0.008874666566650072
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,511,0.010879999647537867
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,511,0.009695999945203463
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,1023,0.011178666104873022
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,1023,0.01101333275437355
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,7,0.008799999952316284
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,2047,0.010965333630641302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,31,0.009557333464423815
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,4095,0.010933333386977514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,8191,0.01121066634853681
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,8191,0.011152000476916632
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,16383,0.015765332927306492
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,32767,0.019215999792019527
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,131071,0.6046293179194132
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,32767,0.0174346665541331
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,65535,0.019424000134070713
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,2047,0.010885333021481832
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,131071,0.05421333511670431
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,131071,0.03743999948104223
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,31,0.008938666433095932
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,1,0.009648000200589498
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,127,0.00933333362142245
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,1,0.009413333609700203
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,16383,0.015168000012636185
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,3,0.010079999764760336
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,1,0.009519999846816063
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,7,0.00997866690158844
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,15,0.010890666395425797
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,15,0.0100853331387043
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,31,0.009610666582981745
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,31,0.01028266673286756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,63,0.010645333677530289
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,4095,0.011007999380429586
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,63,0.010138666878143946
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,127,0.010181333248813948
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,127,0.010714666297038397
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,255,0.009952000031868616
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,511,0.010981333752473196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,1023,0.013573333621025085
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,1023,0.011733333269755045
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,2047,0.029418667157491047
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,2047,0.015466666469971338
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,4095,0.031061333914597828
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,255,0.010922666639089584
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,8191,0.0867786705493927
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,3,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,7,0.009178666397929192
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,16383,0.1591146687666575
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,16383,0.08869333068529765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,4095,0.04884799818197886
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,32767,0.16450132926305136
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,1,0.011152000476916632
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,65535,0.6040746768315634
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,32767,0.3068160017331441
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,1,0.011328000575304031
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,3,0.011136000355084738
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,3,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,65535,0.31519466638565063
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,7,0.011258666714032492
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,15,0.011642667154471079
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,65535,0.03656533360481262
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,31,0.01121066634853681
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,31,0.010853332777818045
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,63,0.011429333438475927
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,7,0.011642667154471079
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,63,0.011018666128317514
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,127,0.011322667201360067
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,511,0.010960000256697336
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,255,0.011968000481526056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,255,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,511,0.012960000584522883
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,15,0.011205332974592844
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,1023,0.02994133283694585
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,1023,0.01594666639963786
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,2047,0.04947733382383982
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,127,0.011109333485364914
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,2047,0.031119999786218006
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,4095,0.05117333432038625
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,511,0.013007999708255133
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,16383,0.3084320028622945
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,16383,0.16835200786590576
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,8191,0.04990399877230326
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,1,0.013072000195582708
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,1,0.015008000036080679
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,4095,0.0872320036093394
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,3,0.013376000026861826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,7,0.013072000195582708
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,8191,0.08803733189900716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,15,0.014394666999578476
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,8191,0.16029333074887595
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,3,0.014794666320085526
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,15,0.013386666774749756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,7,0.015210667004187902
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,31,0.013072000195582708
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,63,0.013242666920026144
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,127,0.01526933287580808
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,127,0.013151999562978745
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,255,0.014954666296641031
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,31,0.014159999787807465
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,255,0.013082666943470636
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,511,0.030031998952229817
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,511,0.01703466723362605
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,63,0.015098666151364645
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,1023,0.04961066444714864
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,1,0.009050666665037474
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,1,0.008863999818762144
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,2047,0.08690667152404785
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,3,0.010672000547250112
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,2047,0.05034666756788889
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,7,0.008992000172535578
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,1023,0.031471999982992806
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,15,0.00980266680320104
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,31,0.009445333232482275
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,3,0.009066666786869368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,4095,0.08797867099444072
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,4095,0.16266666849454245
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,31,0.009189333145817121
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,127,0.009114666531483332
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,255,0.009216000015536943
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,8191,0.3113653262456258
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,8191,0.16355733076731363
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,511,0.009077333534757296
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,15,0.00902399979531765
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,7,0.008954666554927826
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,127,0.009152000149091085
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,2047,0.011087999989589056
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,2047,0.01129066695769628
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,4095,0.011050666371981302
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,255,0.009152000149091085
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,8191,0.013130666067202887
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,8191,0.013258667041858038
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,16383,0.01732800031701724
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,16383,0.015930666277805965
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,511,0.010949333508809408
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,32767,0.019296000401178997
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,1023,0.010911999891201654
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,63,0.008853333070874214
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,1023,0.010922666639089584
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,131071,0.08919999996821086
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,131071,0.053504000107447304
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,4095,0.011029332876205444
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,32767,0.03366933266321818
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,65535,0.05193066596984863
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,63,0.00901333304742972
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,1,0.01918399954835574
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,1,0.01817600056529045
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,3,0.019146667172511418
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,65535,0.03494933247566223
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,7,0.019205333044131596
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,7,0.017445333302021027
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,15,0.019487999379634857
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,3,0.018053332964579265
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,15,0.01720533271630605
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,31,0.01942933350801468
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,31,0.017664000391960144
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,63,0.01926933353145917
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,63,0.017605333278576534
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,127,0.019205333044131596
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,127,0.017349333812793095
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,255,0.029893333713213604
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,255,0.01811733345190684
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,511,0.04850666721661886
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,511,0.03199466566244761
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,1023,0.08566932876904805
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,1023,0.05113600194454193
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,2047,0.15871999661127725
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,2047,0.08877866466840108
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,1,0.03173333406448364
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,4095,0.16461333632469177
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,4095,0.3070560097694397
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,1,0.027445333699385326
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,3,0.027674667537212372
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,3,0.03145066648721695
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,7,0.031386665999889374
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,7,0.02740799884001414
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,15,0.03160000095764796
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,31,0.02889599899450938
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,63,0.027429332335789997
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,63,0.03178666780392329
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,127,0.035429333647092186
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,15,0.027258666853109997
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,127,0.02749866743882497
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,255,0.05043200155099233
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,255,0.03573333223660787
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,511,0.08772266904513042
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,1,0.009029333169261614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,3,0.009039999917149544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,511,0.052757332722345986
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,3,0.00926399976015091
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,7,0.009125333279371262
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,15,0.008965333302815756
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,15,0.009082666908701261
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,7,0.009338666374484697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,31,0.008986666798591614
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,31,0.009317333499590555
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,63,0.009039999917149544
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,1023,0.16048533717791238
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,63,0.009338666374484697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,127,0.009103999783595404
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,31,0.03143999973932902
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,1023,0.09103467067082723
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,255,0.009066666786869368
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,1,0.008885333314538002
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,511,0.010981333752473196
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,1023,0.01098666712641716
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,2047,0.01097600037852923
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,2047,0.01089599976936976
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,255,0.008853333070874214
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,4095,0.012991999586423239
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,8191,0.015301333119471868
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,1023,0.01102399950226148
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,8191,0.015008000036080679
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,16383,0.03204799940188726
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,16383,0.017173333714405697
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,32767,0.03370666752258936
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,511,0.011071999867757162
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,65535,0.08741866548856099
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,65535,0.052298665046691895
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,131071,0.16164799531300864
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,131071,0.09195199608802795
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,2047,0.30824534098307294
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,4095,0.013221333424250284
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,127,0.009290666629870733
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,32767,0.05080533524354299
VLLM,0.16.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,2047,0.16412267088890076
