framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,1,0.011413333316644033
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,31,0.011183999478816986
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,31,0.011125333607196808
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,63,0.01098666712641716
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,1,0.01121066634853681
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,63,0.011018666128317514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,127,0.011098666737476984
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,127,0.010933333386977514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,255,0.011098666737476984
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,255,0.010858666151762009
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,511,0.011120000233252844
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,1023,0.01414399966597557
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,1023,0.01314666618903478
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,2047,0.03011200080315272
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,2047,0.01714666684468587
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,4095,0.048010667165120445
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,4095,0.03142933299144109
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,511,0.011317333827416102
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,1,0.011071999867757162
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,1,0.011002667248249054
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,3,0.011503999431928
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,3,0.01099733387430509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,7,0.01108266661564509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,7,0.010992000500361124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,15,0.011221333096424738
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,3,0.011231999844312668
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,15,0.011007999380429586
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,31,0.011328000575304031
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,31,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,63,0.01097600037852923
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,63,0.010965333630641302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,3,0.011141333729028702
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,127,0.01163200040658315
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,127,0.01139733319481214
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,255,0.01101333275437355
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,255,0.011055999745925268
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,511,0.015066667149464289
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,511,0.01303999995191892
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,1023,0.029648000995318096
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,1023,0.015568000574906668
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,2047,0.04780800143877665
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,4095,0.08471999565760295
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,7,0.01099733387430509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,1,0.008965333302815756
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,4095,0.04993066688378652
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,1,0.008992000172535578
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,3,0.008954666554927826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,3,0.00927466650803884
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,7,0.009125333279371262
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,7,0.009994666402538618
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,15,0.00895999992887179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,15,0.009626666704813639
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,31,0.00901333304742972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,63,0.009077333534757296
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,63,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,127,0.009338666374484697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,127,0.009125333279371262
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,255,0.00915733352303505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,255,0.009130666653315226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,511,0.011039999624093374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,511,0.011007999380429586
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,1023,0.011087999989589056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,1023,0.010879999647537867
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,2047,0.01102399950226148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,2047,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,4095,0.01126933346192042
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,4095,0.01101333275437355
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,8191,0.01714133347074191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,8191,0.01623999948302905
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,16383,0.01915733392039935
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,16383,0.019258666783571243
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,1,0.00898133342464765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,1,0.008863999818762144
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,3,0.010319999729593595
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,3,0.01108266661564509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,7,0.00890666681031386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,15,0.009349333122372627
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,15,0.009093333035707474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,31,0.010015999898314476
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,31,0.009290666629870733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,31,0.009216000015536943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,63,0.009216000015536943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,63,0.009770666559537252
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,127,0.009008000294367472
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,127,0.009306666751702627
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,255,0.00914666677514712
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,255,0.009103999783595404
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,511,0.010015999898314476
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,511,0.009279999881982803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,1023,0.009162666896979014
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,1023,0.01033599985142549
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,2047,0.010949333508809408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,2047,0.011039999624093374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,4095,0.013237333546082178
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,4095,0.012938667088747025
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,8191,0.015125333021084467
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,8191,0.015114666273196539
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,16383,0.01718933383623759
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,16383,0.016986666868130367
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,2047,0.031770666440327965
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,1,0.00891733355820179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,1,0.009306666751702627
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,3,0.009359999870260557
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,7,0.008943999807039896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,3,0.009269333134094873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,7,0.009002666920423508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,7,0.01109333336353302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,15,0.009183999771873156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,15,0.009509333098928133
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,31,0.009056000038981438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,31,0.010805333654085795
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,15,0.011178666104873022
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,127,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,15,0.011157333850860596
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,127,0.011077333241701126
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,255,0.009973333527644476
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,255,0.01109333336353302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,511,0.010970667004585266
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,511,0.011029332876205444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,1023,0.010901333143313726
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,1023,0.010901333143313726
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,2047,0.011237333218256632
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,2047,0.011098666737476984
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,4095,0.015082667271296183
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,4095,0.015103999525308609
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,8191,0.016895999511082966
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,8191,0.01714133347074191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,16383,0.019141333798567455
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,16383,0.018853332847356796
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,1,0.00914666677514712
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,1,0.00922133338948091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,3,0.008954666554927826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,3,0.009056000038981438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,7,0.010159999753038088
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,15,0.008885333314538002
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,15,0.009317333499590555
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,31,0.009082666908701261
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,63,0.00892800030608972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,63,0.009119999905427298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,127,0.009029333169261614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,63,0.008912000184257826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,255,0.009392000113924345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,255,0.00984533317387104
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,511,0.01108266661564509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,511,0.011055999745925268
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,1023,0.01108266661564509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,2047,0.012725333372751871
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,2047,0.013125333935022354
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,4095,0.015066667149464289
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,4095,0.01313599944114685
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,8191,0.017152000218629837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,8191,0.015173333386580149
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,7,0.008938666433095932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,16383,0.017194667210181553
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,16383,0.03145066648721695
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,31,0.008954666554927826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,1,0.012965332716703415
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,1,0.012383999923865
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,3,0.013077333569526672
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,3,0.012655999511480331
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,127,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,7,0.013077333569526672
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,7,0.012389333297808966
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,15,0.013088000317414602
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,7,0.011183999478816986
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,15,0.012634667257467905
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,31,0.013034666577974955
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,31,0.013002666334311167
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,63,0.01313599944114685
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,63,0.012757333616415659
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,127,0.012938667088747025
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,127,0.013088000317414602
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,1023,0.011045332998037338
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,255,0.013114667187134424
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,255,0.013061333447694778
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,511,0.015119999647140503
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,511,0.014991999914248785
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,2047,0.04956266780694326
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,1023,0.031002665559450786
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,2047,0.03317866722742716
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,1023,0.017551999539136887
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,1,0.01393066719174385
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,1,0.013173333058754602
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,3,0.015013333410024643
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,3,0.013749333719412485
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,7,0.013327999661366144
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,15,0.01302933320403099
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,7,0.015072000523408255
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,31,0.01470400020480156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,31,0.012981332838535309
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,15,0.015157333264748255
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,63,0.014032000054915747
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,63,0.015082667271296183
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,127,0.013994666437307993
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,127,0.015125333021084467
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,63,0.009226666763424873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,255,0.01331199953953425
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,255,0.014709333578745524
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,1023,0.04986133178075155
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,511,0.017071999609470367
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,511,0.029738667110602062
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,1023,0.03192000091075897
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,3,0.010768000036478043
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,2047,0.0507893313964208
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,1,0.009045333291093508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,2047,0.08668800195058186
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,7,0.009066666786869368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,3,0.009370666618148485
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,15,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,31,0.010015999898314476
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,31,0.009808000177145004
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,63,0.009077333534757296
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,63,0.010970667004585266
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,127,0.009296000003814697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,127,0.010234666367371878
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,255,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,255,0.01099733387430509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,511,0.011413333316644033
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,511,0.011061333119869232
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,1023,0.011066666493813196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,1023,0.010981333752473196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,2047,0.012965332716703415
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,2047,0.013061333447694778
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,4095,0.015061333775520325
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,4095,0.015146666516860327
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,8191,0.017221332838137943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,8191,0.017008000363906223
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,16383,0.03350933392842611
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,16383,0.021205333371957142
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,1,0.00903466654320558
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,1,0.00914666677514712
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,3,0.009125333279371262
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,3,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,7,0.008976000050703684
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,7,0.00897066667675972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,15,0.008986666798591614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,15,0.009216000015536943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,31,0.008992000172535578
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,31,0.009242666885256767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,63,0.009301333377758661
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,63,0.008901333436369896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,127,0.009242666885256767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,127,0.009109333157539368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,255,0.009279999881982803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,1,0.009946666657924652
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,511,0.010949333508809408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,511,0.01116266722480456
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,7,0.011018666128317514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,15,0.00927466650803884
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,1023,0.011045332998037338
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,1023,0.011098666737476984
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,4095,0.013151999562978745
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,4095,0.015098666151364645
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,2047,0.012698666503032049
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,8191,0.015813333292802174
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,8191,0.03071466585000356
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,16383,0.031504000226656594
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,16383,0.048021331429481506
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,1,0.01637866720557213
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,1,0.015061333775520325
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,3,0.01575999955336253
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,7,0.01515199989080429
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,7,0.016501333564519882
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,15,0.01553600033124288
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,15,0.015141333142916361
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,31,0.016202667107184727
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,31,0.015119999647140503
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,63,0.015306666493415833
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,63,0.01504533365368843
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,255,0.00891733355820179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,127,0.015168000012636185
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,127,0.015381333728631338
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,255,0.015791999797026317
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,255,0.015344000111023584
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,2047,0.013130666067202887
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,511,0.03146666785081228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,511,0.019088000059127808
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,1,0.01926400015751521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,1023,0.05086933573087057
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,1,0.017407999684413273
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,3,0.01929066702723503
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,3,0.01736533393462499
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,3,0.015103999525308609
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,7,0.01913600042462349
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,7,0.01714133347074191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,15,0.017157333592573803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,31,0.01941866676012675
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,31,0.017301333447297413
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,63,0.01918399954835574
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,15,0.019317333896954853
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,63,0.017509333789348602
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,127,0.019333332777023315
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,127,0.017162666966517765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,255,0.017423999806245167
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,1023,0.08628799517949422
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,1023,0.0518506666024526
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,511,0.04906666775544485
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,511,0.031386665999889374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,1023,0.03384533276160558
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,1,0.023317334552605946
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,1,0.01942933350801468
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,3,0.023247999449570973
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,3,0.019189332922299702
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,15,0.01942933350801468
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,7,0.019482667247454327
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,7,0.023589332898457844
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,15,0.023386667172114056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,31,0.02319466571013133
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,255,0.03143466760714849
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,31,0.019466667125622433
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,63,0.023141334454218548
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,63,0.019248000035683315
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,127,0.023258666197458904
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,127,0.019519999623298645
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,255,0.03143999973932902
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,255,0.019813333948453266
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,1,0.02942933390537898
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,1,0.02589866767326991
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,3,0.030805334448814392
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,3,0.025957333544890087
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,7,0.029696000119050343
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,7,0.027386667827765148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,31,0.029792000850041706
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,31,0.027402666707833607
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,15,0.029706666866938274
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,15,0.026021334032217663
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,63,0.027402666707833607
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,63,0.03073599934577942
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,127,0.03342933456103007
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,255,0.05050133168697357
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,255,0.03374933451414108
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,1,0.03818666686614355
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,1,0.031914666295051575
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,3,0.03965866565704346
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,3,0.0330826664964358
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,7,0.03766933331886927
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,15,0.03271999955177307
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,127,0.02773333340883255
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,31,0.03799466788768768
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,7,0.033546666304270424
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,15,0.03819733361403147
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,31,0.03190399954716364
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,63,0.03834133346875509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,63,0.03176533430814743
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,127,0.03997333347797394
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,1,0.045882667104403176
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,127,0.03349866718053818
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,1,0.05373333394527435
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,3,0.05362666646639506
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,3,0.04613866905371348
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,7,0.05418666700522105
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,7,0.04584000011285146
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,15,0.04563199977080027
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,31,0.045882667104403176
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,15,0.053861334919929504
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,31,0.05397333204746246
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,1,0.010602666685978571
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,63,0.05398400127887726
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,7,0.011039999624093374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,3,0.00927466650803884
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,1,0.009301333377758661
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,3,0.01109333336353302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,7,0.011039999624093374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,31,0.009338666374484697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,31,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,63,0.04584000011285146
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,15,0.009637333452701569
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,63,0.009205333267649015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,63,0.011045332998037338
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,127,0.009125333279371262
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,255,0.009301333377758661
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,255,0.009861333295702934
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,127,0.05813866853713989
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,511,0.010869332899649939
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,511,0.011296000331640244
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,127,0.00922133338948091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,1023,0.011274666835864386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,1023,0.011109333485364914
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,2047,0.01322666679819425
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,4095,0.015077333897352219
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,4095,0.015178666760524115
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,8191,0.02993600070476532
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,16383,0.04884799818197886
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,16383,0.03172266731659571
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,2047,0.01309866706530253
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,1,0.009375999992092451
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,1,0.009279999881982803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,127,0.0481333335240682
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,3,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,3,0.009312000125646591
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,7,0.008965333302815756
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,15,0.008992000172535578
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,31,0.009226666763424873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,31,0.009359999870260557
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,63,0.008832000195980072
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,63,0.00914666677514712
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,127,0.009541333342591921
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,127,0.009957333405812582
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,15,0.010933333386977514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,255,0.00898133342464765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,255,0.009189333145817121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,511,0.010949333508809408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,511,0.011066666493813196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,1023,0.013088000317414602
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,1023,0.012053333222866058
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,15,0.009242666885256767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,2047,0.013130666067202887
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,2047,0.013162666310866674
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,4095,0.017157333592573803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,8191,0.04849066833655039
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,8191,0.03148266673088074
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,16383,0.04945066571235657
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,16383,0.08400533596674602
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,8191,0.01724799970785777
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,7,0.009242666885256767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,4095,0.030421334008375805
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,float16,1,0.07055466870466869
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,1,0.05875200033187866
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,float16,3,0.07027199864387512
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,3,0.06006399790445963
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,float16,7,0.06849066913127899
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,7,0.05766933163007101
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,float16,15,0.0705386648575465
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,15,0.06006399790445963
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,float16,31,0.07025066514809926
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,31,0.05909333129723867
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,float16,63,0.06859200199445088
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,63,0.057962665955225624
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,float16,1,0.09644266963005066
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,1,0.08461866776148479
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,3,0.0853760043780009
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,float16,3,0.09567466378211975
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,float16,7,0.09693866968154907
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,7,0.08598933617273967
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,float16,15,0.09734933574994405
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,15,0.08538132905960083
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,31,0.08493866523106892
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,float16,31,0.09683733185132344
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,63,0.08495466907819112
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,float16,63,0.09701866904894511
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,float16,1,0.12870400150616965
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,1,0.10852266351381938
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,float16,3,0.1281013290087382
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,3,0.10839999715487163
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,float16,7,0.12840533256530762
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,7,0.10975999633471172
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,float16,15,0.13010666767756143
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,15,0.1092800001303355
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,float16,31,0.1283466617266337
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,31,0.10822400450706482
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,float16,1,0.1795626680056254
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,1,0.1569866637388865
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,3,0.1586026648680369
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,float16,3,0.1791306734085083
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,1,0.011152000476916632
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,1,0.011109333485364914
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,3,0.010832000523805618
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,float16,7,0.17891732851664224
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,3,0.011061333119869232
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,7,0.15727466344833374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,7,0.009530666594703993
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,7,0.010794666906197866
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,15,0.010314666976531347
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,15,0.011194666226704916
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,31,0.010138666878143946
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,31,0.010928000013033548
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,63,0.0120319997270902
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,63,0.011018666128317514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,127,0.010480000327030817
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,127,0.010981333752473196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,255,0.009786666681369146
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,255,0.011173332730929056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,511,0.011039999624093374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,511,0.011354666203260422
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,1023,0.01301866645614306
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,1023,0.013397333522637686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,2047,0.015168000012636185
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,2047,0.015157333264748255
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,4095,0.03084266682465871
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,4095,0.017818666994571686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,8191,0.048954665660858154
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,8191,0.0322773332397143
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,1,0.009519999846816063
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,1,0.009893333539366722
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,3,0.011215999722480774
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,3,0.010832000523805618
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,7,0.010101333260536194
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,7,0.009296000003814697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,15,0.011205332974592844
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,31,0.01028266673286756
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,31,0.010159999753038088
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,63,0.011029332876205444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,63,0.010960000256697336
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,127,0.01101333275437355
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,float16,15,0.17856534322102866
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,255,0.010869332899649939
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,255,0.00956266683836778
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,511,0.010970667004585266
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,15,0.15708266695340475
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,1023,0.013301332791646322
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,511,0.01126933346192042
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,1023,0.012917333592971167
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,2047,0.029552000264326733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,4095,0.047824000318845115
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,float16,31,0.17880000670750937
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,2047,0.015077333897352219
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,4095,0.030954666435718536
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,31,0.15737600127855936
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,1,0.01097600037852923
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,8191,0.048469334840774536
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,15,0.011098666737476984
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,7,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,7,0.010879999647537867
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,1,0.010970667004585266
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,3,0.010992000500361124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,3,0.010933333386977514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,15,0.011109333485364914
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,15,0.01102399950226148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,127,0.01101333275437355
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,31,0.010901333143313726
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,31,0.011301333705584208
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,63,0.011039999624093374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,63,0.010922666639089584
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,127,0.010933333386977514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,127,0.01108266661564509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,255,0.011045332998037338
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,255,0.03443199892838796
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,511,0.011055999745925268
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,511,0.011488000551859537
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,1023,0.013445333888133367
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,1023,0.01310933381319046
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,2047,0.029333333174387615
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,2047,0.017418666432301205
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,4095,0.047839999198913574
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,3,0.011503999431928
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,3,0.01126933346192042
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,1,0.011071999867757162
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,1,0.01138666644692421
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,7,0.01126933346192042
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,7,0.011253333340088526
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,15,0.011253333340088526
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,31,0.012261333564917246
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,31,0.01109333336353302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,8191,0.0836853285630544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,63,0.011674666156371435
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,63,0.011247999966144562
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,127,0.01137599969903628
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,255,0.01118933285276095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,255,0.01110400011142095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,511,0.015013333410024643
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,1023,0.030000001192092896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,1023,0.01532799998919169
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,2047,0.03145600110292435
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,2047,0.04846400022506714
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,1,0.009285333255926767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,1,0.009114666531483332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,4095,0.04982399940490723
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,3,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,4095,0.031770666440327965
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,3,0.009162666896979014
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,7,0.009312000125646591
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,7,0.008986666798591614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,15,0.009290666629870733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,15,0.00933333362142245
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,31,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,31,0.009050666665037474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,63,0.009152000149091085
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,63,0.008949333180983862
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,127,0.008954666554927826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,127,0.010762666662534079
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,15,0.011098666737476984
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,255,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,511,0.01091733326514562
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,511,0.01110400011142095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,1023,0.01102399950226148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,1023,0.011002667248249054
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,2047,0.010933333386977514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,2047,0.010874666273593903
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,4095,0.011130666981140772
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,127,0.01108266661564509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,8191,0.01709866647919019
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,8191,0.01714666684468587
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,16383,0.019296000401178997
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,16383,0.019093333433071773
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,511,0.012986666212479273
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,1,0.008885333314538002
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,1,0.009072000160813332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,3,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,3,0.009109333157539368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,7,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,15,0.008997333546479544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,7,0.009242666885256767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,15,0.008933333059151968
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,31,0.008943999807039896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,63,0.009008000294367472
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,127,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,4095,0.08425600330034892
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,127,0.009418666362762451
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,255,0.009039999917149544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,511,0.011071999867757162
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,511,0.01081066702802976
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,1023,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,1023,0.011039999624093374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,2047,0.010922666639089584
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,255,0.009882666791478792
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,4095,0.01332266628742218
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,4095,0.01293333371480306
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,4095,0.011157333850860596
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,8191,0.015119999647140503
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,8191,0.015205333630243937
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,16383,0.01735466718673706
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,16383,0.01681600014368693
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,1,0.009242666885256767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,1,0.009194666519761086
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,3,0.008874666566650072
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,3,0.009674666449427605
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,7,0.008997333546479544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,7,0.009173333023985228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,15,0.009290666629870733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,15,0.009685333197315535
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,31,0.008922666932145754
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,31,0.008997333546479544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,31,0.009242666885256767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,63,0.009898666913310686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,63,0.00903466654320558
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,63,0.008933333059151968
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,127,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,127,0.00961599995692571
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,255,0.010602666685978571
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,511,0.010922666639089584
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,511,0.011002667248249054
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,255,0.00898133342464765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,1023,0.011007999380429586
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,2047,0.011045332998037338
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,2047,0.01097600037852923
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,4095,0.015290666371583939
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,4095,0.01515199989080429
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,8191,0.016821333517630894
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,8191,0.01703466723362605
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,16383,0.019199999670187633
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,16383,0.017184000462293625
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,2047,0.011007999380429586
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,1,0.009290666629870733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,1,0.00892800030608972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,3,0.008986666798591614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,3,0.008938666433095932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,7,0.009397333487868309
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,7,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,15,0.008863999818762144
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,31,0.009290666629870733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,31,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,63,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,63,0.009343999748428663
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,127,0.009114666531483332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,127,0.009482666850090027
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,255,0.009882666791478792
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,255,0.009050666665037474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,511,0.010858666151762009
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,511,0.010960000256697336
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,1023,0.011029332876205444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,1023,0.011002667248249054
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,2047,0.012597333639860153
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,4095,0.015125333021084467
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,4095,0.013616000612576803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,8191,0.017077332983414333
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,8191,0.015125333021084467
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,16383,0.017935999979575474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,16383,0.033039999504884086
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,1,0.012858666479587555
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,255,0.01108266661564509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,1,0.014218666901191076
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,3,0.013082666943470636
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,3,0.013061333447694778
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,7,0.012975999464591345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,7,0.013061333447694778
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,15,0.011440000186363855
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,31,0.01209066684047381
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,31,0.012975999464591345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,63,0.013114667187134424
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,63,0.012400000045696894
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,15,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,127,0.013034666577974955
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,127,0.013050666699806849
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,255,0.013077333569526672
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,255,0.012506666282812754
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,511,0.014831999937693277
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,1023,0.031498665610949196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,2047,0.01184533288081487
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,1023,0.017727999637524288
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,2047,0.04986133178075155
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,2047,0.03316266586383184
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,1,0.01331199953953425
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,3,0.01322666679819425
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,1,0.015072000523408255
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,7,0.015178666760524115
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,3,0.015098666151364645
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,1023,0.010981333752473196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,7,0.01333333303531011
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,15,0.01516266663869222
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,15,0.012997332960367203
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,15,0.013354666531085968
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,31,0.015098666151364645
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,63,0.01331199953953425
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,127,0.015285332997639975
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,127,0.013023999830087027
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,255,0.014949332922697067
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,255,0.014298666268587112
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,511,0.015103999525308609
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,511,0.01708799973130226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,511,0.030821333328882854
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,1023,0.03170666595300039
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,1023,0.04820266862710317
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,1,0.009258666386206945
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,1,0.00897066667675972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,3,0.009322666873534521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,3,0.009472000102202097
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,2047,0.08649067083994548
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,2047,0.04983466863632202
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,7,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,15,0.009050666665037474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,15,0.008943999807039896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,31,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,31,0.009216000015536943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,63,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,63,0.009189333145817121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,127,0.009109333157539368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,127,0.008949333180983862
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,255,0.009306666751702627
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,255,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,511,0.011018666128317514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,511,0.01099733387430509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,1023,0.01089599976936976
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,31,0.013471999516089758
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,1023,0.010992000500361124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,2047,0.01321600005030632
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,63,0.015077333897352219
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,2047,0.013295999417702356
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,4095,0.015034666905800501
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,4095,0.01509333277742068
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,8191,0.018207999567190807
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,8191,0.017130666722853977
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,16383,0.0334346666932106
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,16383,0.021173333128293354
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,1,0.00890666681031386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,1,0.009354666496316591
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,3,0.009141333401203156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,3,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,7,0.009888000165422758
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,7,0.009002666920423508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,15,0.008933333059151968
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,31,0.009029333169261614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,31,0.008992000172535578
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,63,0.00960533320903778
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,127,0.009269333134094873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,127,0.009136000027259191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,255,0.00898133342464765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,255,0.010064000263810158
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,511,0.01102399950226148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,511,0.010933333386977514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,1023,0.01090666651725769
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,1023,0.011146667102972666
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,2047,0.013007999708255133
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,2047,0.011877333124478659
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,4095,0.015061333775520325
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,8191,0.030591999491055805
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,8191,0.017194667210181553
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,7,0.010842667271693548
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,16383,0.0481333335240682
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,16383,0.03107733279466629
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,1,0.015541333705186844
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,1,0.015141333142916361
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,3,0.015461333096027374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,3,0.015082667271296183
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,7,0.015450666348139444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,7,0.015135999768972397
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,15,0.015413332730531693
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,15,0.015178666760524115
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,31,0.016805333395799
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,31,0.015072000523408255
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,15,0.009136000027259191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,63,0.009285333255926767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,63,0.015317333241303762
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,63,0.015050667027632395
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,127,0.01534933348496755
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,127,0.015077333897352219
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,255,0.015824000040690105
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,255,0.01504533365368843
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,4095,0.013056000073750814
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,511,0.0314026673634847
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,511,0.019296000401178997
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,1,0.01911466692884763
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,1,0.017877332866191864
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,3,0.019146667172511418
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,3,0.017450666675964992
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,1023,0.03341866781314214
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,1023,0.04986133178075155
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,7,0.01937066639463107
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,15,0.01911466692884763
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,15,0.017386666188637417
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,31,0.017423999806245167
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,31,0.01921066641807556
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,7,0.01724799970785777
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,127,0.019285333653291065
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,127,0.01739199956258138
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,255,0.03054933249950409
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,63,0.01807466646035512
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,255,0.017114666601022083
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,63,0.019424000134070713
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,1023,0.08530666430791219
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,511,0.049626668294270836
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,1023,0.05029866596062978
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,1,0.0232640008131663
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,3,0.023311999936898548
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,1,0.0194560003777345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,3,0.019487999379634857
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,7,0.019424000134070713
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,15,0.02332799881696701
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,15,0.019472000499566395
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,7,0.023455999791622162
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,31,0.023258666197458904
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,31,0.019498666127522785
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,63,0.023317334552605946
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,63,0.019226666539907455
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,127,0.023376000424226124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,511,0.03186133255561193
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,127,0.0198186660806338
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,1,0.02934933453798294
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,255,0.031685332457224526
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,255,0.020165332903464634
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,1,0.025370667378107708
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,3,0.02940800040960312
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,3,0.02739199995994568
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,7,0.0272533322374026
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,15,0.02942399928967158
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,31,0.027072000006834667
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,31,0.029722665747006733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,7,0.029706666866938274
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,15,0.02571200082699458
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,63,0.029706666866938274
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,63,0.02733866622050603
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,127,0.0335413341720899
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,127,0.026144000391165417
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,255,0.03282133241494497
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,1,0.03763733307520548
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,1,0.03159466634194056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,3,0.03349333256483078
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,3,0.039264000952243805
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,7,0.037845333417256675
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,7,0.03272533416748047
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,15,0.03793599953254064
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,15,0.031930667658646904
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,255,0.049813335140546165
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,31,0.037632000943024956
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,31,0.03209600100914637
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,63,0.037685332198937736
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,63,0.03212266663710276
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,127,0.0402453343073527
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,127,0.03363200028737386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,1,0.0539626677831014
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,1,0.04577599962552389
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,3,0.05413866539796194
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,3,0.045824001232783
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,7,0.053957333167394005
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,7,0.04576533536116282
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,15,0.05400000015894572
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,1,0.01055466632048289
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,15,0.04594666759173075
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,1,0.009919999788204828
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,3,0.009056000038981438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,3,0.010346666599313417
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,31,0.056661332647005715
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,31,0.04540266593297323
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,7,0.009045333291093508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,7,0.010559999694426855
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,31,0.009269333134094873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,63,0.05421866476535797
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,63,0.04600533346335093
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,31,0.009935999910036722
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,15,0.00892800030608972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,15,0.01108266661564509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,63,0.00927466650803884
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,127,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,255,0.009413333609700203
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,255,0.010853332777818045
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,511,0.011087999989589056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,511,0.01118933285276095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,1023,0.011546666423479715
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,63,0.010911999891201654
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,1023,0.012096000214417776
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,127,0.04774933556715647
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,127,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,2047,0.01301866645614306
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,4095,0.015114666273196539
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,4095,0.014954666296641031
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,8191,0.03143999973932902
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,8191,0.01710933322707812
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,16383,0.03149333347876867
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,1,0.009093333035707474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,1,0.00897066667675972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,3,0.009349333122372627
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,3,0.00922133338948091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,16383,0.048538664976755776
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,7,0.009242666885256767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,7,0.009082666908701261
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,15,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,15,0.009008000294367472
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,31,0.010992000500361124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,31,0.00922133338948091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,63,0.009018666421373686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,127,0.010970667004585266
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,255,0.009130666653315226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,255,0.011007999380429586
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,511,0.01101333275437355
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,1023,0.013082666943470636
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,1023,0.012703999876976013
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,2047,0.014901333798964819
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,2047,0.012991999586423239
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,4095,0.03046933313210805
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,127,0.057946667075157166
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,4095,0.017018667111794155
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,2047,0.012981332838535309
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,8191,0.048325334986050926
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,8191,0.03295466552178065
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,16383,0.04925866425037384
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,16383,0.08475200335184734
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,63,0.008965333302815756
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,127,0.008986666798591614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,511,0.010992000500361124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,1,0.056159997979799904
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,float16,1,0.06843733290831248
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,float16,3,0.06970666845639546
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,3,0.05834666887919108
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,float16,7,0.07025066514809926
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,7,0.05807999769846598
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,float16,15,0.07035733262697856
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,15,0.058133333921432495
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,float16,31,0.06859733164310455
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,31,0.05753066639105479
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,float16,63,0.06860266625881195
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,1,0.08475200335184734
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,63,0.05840000013510386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,float16,3,0.09699199597040813
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,3,0.08469333251317342
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,float16,7,0.09699199597040813
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,7,0.08538666367530823
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,float16,1,0.0963200032711029
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,float16,15,0.09713600079218547
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,15,0.0849226713180542
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,float16,31,0.09704533219337463
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,31,0.08504000306129456
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,float16,63,0.096261332432429
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,63,0.08528533577919006
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,1,0.10733866691589355
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,float16,3,0.12690132856369019
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,3,0.10891733566919963
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,7,0.10845866799354553
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,float16,15,0.1283519963423411
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,15,0.1092639962832133
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,float16,1,0.12572800119717917
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,float16,7,0.12716799974441528
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,float16,31,0.1290773351987203
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,float16,1,0.17868266503016153
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,31,0.10725866754849751
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,1,0.1569813291231791
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,float16,3,0.17882132530212402
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,1,0.01032533310353756
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,1,0.010960000256697336
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,3,0.010234666367371878
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,3,0.010954666882753372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,3,0.15888532996177673
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,7,0.009125333279371262
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,15,0.00915733352303505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,7,0.011039999624093374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,15,0.009722666814923286
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,31,0.010992000500361124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,31,0.010064000263810158
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,63,0.010453333457310995
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,127,0.009322666873534521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,127,0.010037333394090334
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,float16,7,0.17887999614079794
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,255,0.009349333122372627
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,255,0.011050666371981302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,511,0.011242666592200598
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,7,0.15743999679883322
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,511,0.010901333143313726
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,1023,0.012954667210578918
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,1023,0.012965332716703415
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,2047,0.015109332899252573
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,2047,0.015200000256299973
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,4095,0.031471999982992806
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,float16,15,0.17951999107996622
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,4095,0.017535999417304993
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,8191,0.048895999789237976
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,8191,0.03239466746648153
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,15,0.15852800011634827
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,1,0.011050666371981302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,float16,31,0.17941866318384805
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,3,0.01108266661564509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,3,0.01118933285276095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,1,0.009317333499590555
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,7,0.010901333143313726
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,15,0.010309333602587381
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,15,0.011658667276302973
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,31,0.011109333485364914
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,7,0.011253333340088526
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,63,0.01110400011142095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,63,0.010591999938090643
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,31,0.011029332876205444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,127,0.010640000303586325
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,127,0.01091733326514562
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,255,0.01089599976936976
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,255,0.010944000134865442
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,1023,0.01322666679819425
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,511,0.011301333705584208
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,511,0.010981333752473196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,1023,0.013002666334311167
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,2047,0.029487999776999157
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,2047,0.015360000232855478
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,4095,0.031066666046778362
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,4095,0.047775998711586
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,8191,0.08391466736793518
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,8191,0.047983999053637184
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,1,0.010944000134865442
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,63,0.009872000043590864
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,1,0.011146667102972666
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,3,0.011039999624093374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,3,0.011061333119869232
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,7,0.011071999867757162
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,15,0.010837333897749582
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,31,0.01101333275437355
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,31,0.01089599976936976
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,63,0.01097600037852923
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,63,0.010933333386977514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,31,0.1569546659787496
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,127,0.01089599976936976
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,127,0.011050666371981302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,255,0.010933333386977514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,255,0.010960000256697336
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,511,0.011247999966144562
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,511,0.011183999478816986
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,1023,0.013343999783198038
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,1023,0.012906666845083237
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,2047,0.01736533393462499
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,4095,0.03160533308982849
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,4095,0.04781333108743032
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,1,0.011087999989589056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,2047,0.029333333174387615
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,3,0.012293333808581034
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,3,0.01102399950226148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,7,0.011125333607196808
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,7,0.01126933346192042
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,15,0.011231999844312668
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,15,0.011039999624093374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,31,0.011050666371981302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,31,0.010842667271693548
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,63,0.011391999820868174
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,63,0.011221333096424738
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,127,0.011061333119869232
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,127,0.01099733387430509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,7,0.011152000476916632
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,255,0.01098666712641716
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,255,0.011264000087976456
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,15,0.010933333386977514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,511,0.014474666366974512
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,1023,0.015471999843915304
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,1023,0.029674666623274486
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,2047,0.047744000951449074
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,2047,0.03143466760714849
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,1,0.01091733326514562
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,4095,0.08474666873613994
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,3,0.009189333145817121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,7,0.01044800008336703
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,4095,0.04991999765237173
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,7,0.009338666374484697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,15,0.008949333180983862
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,15,0.010938666760921478
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,31,0.009162666896979014
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,31,0.009946666657924652
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,63,0.009114666531483332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,63,0.009941333283980688
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,127,0.009765333185593287
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,127,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,255,0.00926399976015091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,255,0.010901333143313726
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,511,0.01108266661564509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,1,0.010992000500361124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,1023,0.011007999380429586
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,1023,0.010842667271693548
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,2047,0.011050666371981302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,2047,0.010933333386977514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,4095,0.011359999577204386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,4095,0.011215999722480774
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,8191,0.017018667111794155
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,8191,0.017125333348910015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,16383,0.01934933289885521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,16383,0.019167999426523846
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,1,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,1,0.009306666751702627
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,3,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,3,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,7,0.009061333412925402
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,7,0.008976000050703684
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,15,0.009808000177145004
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,15,0.009434666484594345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,31,0.009317333499590555
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,511,0.01310933381319046
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,31,0.009178666397929192
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,63,0.00892800030608972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,63,0.010255999863147736
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,1,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,127,0.01055466632048289
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,127,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,255,0.009189333145817121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,255,0.009008000294367472
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,3,0.00891733355820179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,511,0.010949333508809408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,511,0.011007999380429586
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,1023,0.010858666151762009
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,1023,0.01089599976936976
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,2047,0.011087999989589056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,2047,0.010901333143313726
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,4095,0.013253333667914072
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,4095,0.013045333325862885
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,8191,0.015103999525308609
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,8191,0.015114666273196539
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,1,0.009114666531483332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,16383,0.017477333545684814
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,16383,0.017210666090250015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,3,0.008922666932145754
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,511,0.011258666714032492
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,7,0.00914666677514712
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,7,0.009599999835093817
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,15,0.009632000078757605
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,15,0.009205333267649015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,31,0.009354666496316591
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,31,0.00914666677514712
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,63,0.00895999992887179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,63,0.00895999992887179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,127,0.009621333330869675
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,127,0.011066666493813196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,255,0.00922133338948091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,255,0.008976000050703684
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,511,0.010954666882753372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,511,0.011770666887362799
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,1023,0.01109333336353302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,2047,0.010965333630641302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,2047,0.010922666639089584
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,4095,0.015034666905800501
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,8191,0.01710933322707812
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,8191,0.015226667126019796
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,16383,0.019317333896954853
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,16383,0.018170667191346485
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,1,0.008949333180983862
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,1,0.00922133338948091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,3,0.009119999905427298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,3,0.009093333035707474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,7,0.00926399976015091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,7,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,15,0.00915733352303505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,15,0.009338666374484697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,31,0.00914666677514712
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,31,0.009328000247478485
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,63,0.00897066667675972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,63,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,127,0.009279999881982803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,1,0.010933333386977514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,255,0.009957333405812582
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,3,0.009173333023985228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,255,0.011989332735538483
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,511,0.01097600037852923
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,511,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,1023,0.01166933278242747
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,1023,0.011039999624093374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,2047,0.012335999558369318
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,2047,0.012944000462690989
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,4095,0.015109332899252573
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,4095,0.013264000415802002
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,8191,0.014991999914248785
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,8191,0.01711999997496605
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,1023,0.011152000476916632
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,16383,0.01876266673207283
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,1,0.013056000073750814
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,4095,0.014746667196353277
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,1,0.011727999895811081
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,3,0.013007999708255133
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,3,0.012757333616415659
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,7,0.013114667187134424
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,15,0.012479999413092932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,7,0.013050666699806849
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,31,0.013050666699806849
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,31,0.012453333785136541
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,63,0.012975999464591345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,63,0.01303999995191892
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,127,0.012970666090647379
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,127,0.012613333761692047
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,255,0.013199999928474426
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,255,0.011936000237862269
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,511,0.01515199989080429
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,511,0.01492799942692121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,127,0.009392000113924345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,1023,0.030762667457262676
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,2047,0.04921066761016846
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,1023,0.017221332838137943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,2047,0.03232000023126602
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,1,0.01509333277742068
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,3,0.014058666924635569
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,1,0.013408000270525614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,16383,0.032399999598662056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,3,0.013359999905029932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,7,0.01309866706530253
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,7,0.014890667051076889
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,15,0.013306666165590286
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,15,0.014725333700577417
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,31,0.014981333166360855
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,31,0.013386666774749756
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,63,0.013487999637921652
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,15,0.013023999830087027
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,127,0.013072000195582708
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,127,0.015168000012636185
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,255,0.015072000523408255
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,255,0.014042666802803675
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,511,0.030837332208951313
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,1023,0.0496373325586319
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,1,0.009365333244204521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,2047,0.05163733164469401
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,2047,0.08558932940165202
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,1,0.009690666571259499
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,3,0.009077333534757296
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,3,0.009141333401203156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,7,0.009872000043590864
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,7,0.010981333752473196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,15,0.009296000003814697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,15,0.009162666896979014
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,31,0.009519999846816063
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,31,0.010960000256697336
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,63,0.009893333539366722
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,63,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,127,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,127,0.008986666798591614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,255,0.009525333220760027
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,255,0.009072000160813332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,511,0.010901333143313726
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,511,0.010949333508809408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,1023,0.011029332876205444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,1023,0.011242666592200598
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,2047,0.013050666699806849
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,2047,0.012970666090647379
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,4095,0.015146666516860327
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,4095,0.015370666980743408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,63,0.015040000279744467
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,8191,0.017210666090250015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,16383,0.03362133353948593
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,16383,0.021407999098300934
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,1,0.009258666386206945
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,1,0.009194666519761086
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,3,0.009072000160813332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,3,0.009178666397929192
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,7,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,7,0.009279999881982803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,15,0.008943999807039896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,15,0.009568000212311745
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,511,0.017082666357358296
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,31,0.009269333134094873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,31,0.009205333267649015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,63,0.009872000043590864
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,63,0.008992000172535578
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,127,0.009226666763424873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,255,0.009050666665037474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,255,0.009765333185593287
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,511,0.011077333241701126
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,127,0.009061333412925402
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,1023,0.01099733387430509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,1023,0.010911999891201654
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,2047,0.01321600005030632
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,2047,0.012698666503032049
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,4095,0.015072000523408255
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,4095,0.01333333303531011
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,8191,0.02935466667016347
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,8191,0.016927999754746754
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,16383,0.03139200061559677
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,1,0.0161013330022494
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,1,0.015087999403476715
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,3,0.015397333850463232
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,8191,0.017157333592573803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,3,0.015168000012636185
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,7,0.015423999478419622
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,7,0.015109332899252573
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,15,0.015418666104475657
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,15,0.014965333044528961
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,31,0.015728000551462173
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,1023,0.03190399954716364
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,31,0.015210667004187902
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,63,0.015386667102575302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,63,0.015125333021084467
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,127,0.01504533365368843
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,511,0.010714666297038397
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,127,0.015098666151364645
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,255,0.016437333077192307
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,255,0.015141333142916361
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,16383,0.04822933177153269
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,511,0.03141333411137263
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,1,0.01915733392039935
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,1,0.01709866647919019
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,511,0.01923199991385142
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,1023,0.04985600213209788
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,1023,0.03346666693687439
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,7,0.019482667247454327
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,3,0.01714133347074191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,7,0.017152000218629837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,15,0.019472000499566395
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,31,0.01710933322707812
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,63,0.01724799970785777
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,63,0.019194666296243668
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,15,0.01736533393462499
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,31,0.019258666783571243
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,127,0.017157333592573803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,255,0.018085333208243053
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,255,0.030693332354227703
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,3,0.019152000546455383
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,1023,0.05020800232887268
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,511,0.03219733387231827
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,511,0.0498933345079422
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,1023,0.088128000497818
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,1,0.023232000569502514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,1,0.019237333287795384
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,3,0.02362666775782903
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,3,0.01916266605257988
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,15,0.023189333577950794
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,15,0.019866666446129482
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,7,0.02333866556485494
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,7,0.019509332875410717
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,31,0.023210667073726654
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,127,0.0194560003777345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,31,0.019850666324297588
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,63,0.023333333432674408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,63,0.019930666933457058
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,127,0.023221333821614582
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,127,0.01979200045267741
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,255,0.03164800008138021
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,1,0.030426666140556335
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,255,0.01951466624935468
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,1,0.025626666843891144
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,3,0.02940266579389572
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,3,0.025642665723959606
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,7,0.029711998999118805
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,15,0.025605333348115284
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,7,0.025973332424958546
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,31,0.026687999566396076
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,31,0.02942933390537898
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,15,0.029440000653266907
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,63,0.029685333371162415
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,63,0.025578667720158894
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,127,0.033520000676314034
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,127,0.025941332181294758
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,255,0.0335359995563825
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,255,0.04991999765237173
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,1,0.03859733293453852
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,3,0.037632000943024956
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,3,0.031701333820819855
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,7,0.038021333515644073
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,7,0.03173866619666418
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,31,0.037861332297325134
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,15,0.03764266769091288
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,15,0.03146666785081228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,31,0.03170666595300039
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,63,0.03745066622893015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,1,0.0317493329445521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,63,0.03173333406448364
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,1,0.04478933413823446
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,1,0.05398933092753092
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,127,0.032416000962257385
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,127,0.03976533313592275
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,7,0.053930665055910744
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,3,0.04577066500981649
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,3,0.054858664671579994
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,7,0.04587733248869578
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,15,0.04552533229192098
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,31,0.054058666030565895
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,1,0.009328000247478485
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,31,0.04590400060017904
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,1,0.010575999816258749
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,3,0.009029333169261614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,7,0.009029333169261614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,63,0.05417599777380625
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,3,0.009136000027259191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,7,0.00956266683836778
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,15,0.009194666519761086
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,31,0.009290666629870733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,63,0.008943999807039896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,63,0.009018666421373686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,127,0.008954666554927826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,127,0.010965333630641302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,127,0.047824000318845115
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,255,0.008879999940594038
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,255,0.01091733326514562
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,511,0.01102399950226148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,1023,0.011359999577204386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,1023,0.011055999745925268
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,511,0.01110400011142095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,2047,0.01331199953953425
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,2047,0.013045333325862885
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,4095,0.015114666273196539
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,15,0.05369600156943003
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,4095,0.015114666273196539
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,8191,0.029743999242782593
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,8191,0.017504000415404636
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,16383,0.048063998421033226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,16383,0.032111999889214836
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,1,0.008976000050703684
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,1,0.009290666629870733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,3,0.008943999807039896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,63,0.04587733248869578
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,3,0.009893333539366722
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,7,0.009285333255926767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,7,0.009269333134094873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,127,0.05731200178464254
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,15,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,15,0.011120000233252844
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,31,0.00922133338948091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,31,0.00915733352303505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,63,0.009216000015536943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,15,0.009375999992092451
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,127,0.009226666763424873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,127,0.00997866690158844
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,255,0.009018666421373686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,255,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,511,0.01089599976936976
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,63,0.009674666449427605
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,1023,0.012960000584522883
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,511,0.01121066634853681
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,1023,0.011253333340088526
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,2047,0.01413333291808764
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,2047,0.013290667285521826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,4095,0.029680001238981884
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,8191,0.03134933362404505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,8191,0.0481333335240682
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,16383,0.04945066571235657
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,16383,0.08469866712888081
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,31,0.009743999689817429
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,4095,0.015333333363135656
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,float16,1,0.06921066840489705
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,1,0.05608533322811127
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,float16,7,0.07047466437021892
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,3,0.05813866853713989
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,float16,3,0.06941866874694824
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,7,0.05603733162085215
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,float16,15,0.07060266534487407
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,15,0.057999998331069946
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,float16,31,0.06883733471234639
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,31,0.05682133138179779
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,float16,63,0.0702453354994456
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,float16,1,0.09636267026265462
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,63,0.0580320010582606
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,1,0.08476799726486206
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,3,0.08577066659927368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,float16,3,0.09629333019256592
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,float16,7,0.09628799557685852
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,7,0.08475733796755473
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,float16,15,0.09620799620946248
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,float16,31,0.0969546635945638
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,15,0.0849226713180542
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,31,0.08479467034339905
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,float16,63,0.09737066427866618
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,63,0.08499200145403545
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,float16,1,0.1269706686337789
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,1,0.10725866754849751
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,float16,3,0.12778666615486145
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,3,0.108624001344045
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,float16,7,0.1269653340180715
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,7,0.10823999842007954
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,float16,15,0.1288693348566691
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,15,0.10814399520556132
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,31,0.1072106659412384
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,float16,1,0.178165336449941
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,float16,31,0.12574399511019388
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,1,0.1574613352616628
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,3,0.15709333618481955
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,float16,3,0.18073066075642905
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,1,0.010863999525705973
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,3,0.011077333241701126
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,1,0.010874666273593903
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,3,0.011018666128317514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,7,0.010058666889866194
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,7,0.010949333508809408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,15,0.011130666981140772
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,15,0.01108266661564509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,31,0.011136000355084738
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,31,0.01091733326514562
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,63,0.010858666151762009
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,63,0.01101333275437355
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,float16,7,0.17850132783253989
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,127,0.011120000233252844
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,127,0.010954666882753372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,255,0.011114666859308878
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,511,0.011029332876205444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,255,0.010591999938090643
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,511,0.01157333329319954
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,1023,0.01301866645614306
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,1023,0.012997332960367203
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,2047,0.015226667126019796
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,2047,0.015066667149464289
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,4095,0.03143466760714849
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,4095,0.017173333714405697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,8191,0.04886400202910105
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,8191,0.032469332218170166
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,1,0.010890666395425797
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,7,0.15662399927775064
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,1,0.010853332777818045
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,3,0.011045332998037338
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,3,0.011087999989589056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,7,0.00915733352303505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,7,0.009957333405812582
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,15,0.010693332801262537
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,15,0.00966933307548364
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,31,0.010741333166758219
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,31,0.009306666751702627
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,63,0.009626666704813639
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,63,0.010714666297038397
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,127,0.010863999525705973
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,127,0.009610666582981745
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,255,0.01110400011142095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,255,0.009482666850090027
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,511,0.010992000500361124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,511,0.011039999624093374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,15,0.15667733550071716
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,1023,0.013088000317414602
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,float16,15,0.17802133162816366
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,1023,0.012981332838535309
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,2047,0.029482667644818623
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,2047,0.01505600040157636
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,4095,0.047781333327293396
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,31,0.1585546632607778
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,8191,0.0488373339176178
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,4095,0.03075733284155528
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,1,0.009312000125646591
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,8191,0.08329066634178162
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,3,0.011183999478816986
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,7,0.009258666386206945
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,7,0.010981333752473196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,15,0.009285333255926767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,31,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,31,0.010954666882753372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,3,0.010901333143313726
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,63,0.009183999771873156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,63,0.009296000003814697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,127,0.010981333752473196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,127,0.011018666128317514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,255,0.009402666861812273
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,255,0.011061333119869232
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,511,0.01098666712641716
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,511,0.01110400011142095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,1023,0.01310933381319046
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,1023,0.01310933381319046
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,2047,0.015119999647140503
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,2047,0.015098666151364645
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,4095,0.031328000128269196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,4095,0.017184000462293625
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,8191,0.04980266590913137
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,8191,0.03345600018898646
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,1,0.010026666646202406
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,3,0.010928000013033548
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,1,0.011050666371981302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,3,0.011055999745925268
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,float16,31,0.17991999785105386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,7,0.010938666760921478
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,7,0.011071999867757162
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,1,0.00914666677514712
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,15,0.009402666861812273
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,31,0.010064000263810158
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,31,0.01089599976936976
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,63,0.01081066702802976
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,127,0.010970667004585266
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,255,0.01007466639081637
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,255,0.009557333464423815
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,15,0.010928000013033548
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,511,0.01119999960064888
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,511,0.011141333729028702
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,2047,0.029546665648619335
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,1023,0.01310933381319046
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,1023,0.01301866645614306
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,2047,0.015114666273196539
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,4095,0.04788800080617269
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,4095,0.031343999008337654
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,8191,0.0488373339176178
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,8191,0.08469866712888081
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,1,0.01102399950226148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,3,0.011760000139474869
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,3,0.011146667102972666
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,7,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,7,0.011061333119869232
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,15,0.01192533348997434
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,15,0.011242666592200598
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,31,0.01102399950226148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,15,0.010618666807810465
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,63,0.011359999577204386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,63,0.011285333583752314
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,127,0.01129066695769628
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,63,0.011125333607196808
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,255,0.011050666371981302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,255,0.011098666737476984
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,511,0.013653332988421122
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,511,0.01314666618903478
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,127,0.010890666395425797
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,1023,0.029605334003766377
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,1023,0.01515199989080429
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,2047,0.03140799949566523
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,2047,0.04844800134499868
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,4095,0.0848640004793803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,4095,0.049738665421803795
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,1,0.01108266661564509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,1,0.00901333304742972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,8191,0.15742933750152588
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,8191,0.0870293378829956
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,3,0.009279999881982803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,3,0.009269333134094873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,7,0.008922666932145754
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,15,0.009018666421373686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,31,0.009098666409651438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,31,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,31,0.010938666760921478
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,63,0.009162666896979014
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,63,0.010981333752473196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,127,0.00972800018886725
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,127,0.01099733387430509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,255,0.01108266661564509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,511,0.010928000013033548
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,127,0.011050666371981302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,511,0.011178666104873022
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,1023,0.011087999989589056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,1023,0.010981333752473196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,2047,0.011109333485364914
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,2047,0.010954666882753372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,4095,0.010981333752473196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,4095,0.011594666788975397
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,8191,0.017050666113694508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,8191,0.015333333363135656
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,16383,0.020432000358899433
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,16383,0.021269333859284718
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,32767,0.023567999402681988
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,32767,0.023413332800070446
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,1,0.008890666688481966
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,1,0.00922133338948091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,3,0.009178666397929192
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,7,0.00938666673998038
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,7,0.009152000149091085
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,1,0.009242666885256767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,15,0.00916800027092298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,15,0.008943999807039896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,31,0.009162666896979014
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,63,0.009066666786869368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,7,0.009098666409651438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,63,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,15,0.009136000027259191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,127,0.00890666681031386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,255,0.009050666665037474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,255,0.009173333023985228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,511,0.009354666496316591
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,1023,0.010784000158309937
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,1023,0.009141333401203156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,255,0.009306666751702627
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,2047,0.009269333134094873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,2047,0.010949333508809408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,4095,0.01110400011142095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,8191,0.015311999867359797
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,8191,0.015066667149464289
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,16383,0.017850667238235474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,16383,0.01724799970785777
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,32767,0.021173333128293354
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,32767,0.019386666516462963
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,1,0.008896000062425932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,1,0.009103999783595404
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,3,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,7,0.009205333267649015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,7,0.009152000149091085
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,3,0.009119999905427298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,15,0.008954666554927826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,15,0.009072000160813332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,31,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,63,0.009119999905427298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,31,0.009130666653315226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,31,0.008954666554927826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,63,0.009226666763424873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,127,0.009130666653315226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,127,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,255,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,255,0.008992000172535578
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,511,0.011152000476916632
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,511,0.010992000500361124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,1023,0.009290666629870733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,1023,0.010890666395425797
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,2047,0.01108266661564509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,4095,0.011205332974592844
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,2047,0.010805333654085795
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,4095,0.013189333180586496
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,4095,0.013327999661366144
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,8191,0.015114666273196539
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,8191,0.015237333873907724
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,16383,0.016442666451136272
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,16383,0.017488000293572743
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,32767,0.03342399994532267
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,1,0.009418666362762451
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,32767,0.019274666905403137
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,1,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,3,0.009285333255926767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,7,0.008992000172535578
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,7,0.009349333122372627
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,3,0.008943999807039896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,15,0.009103999783595404
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,15,0.009119999905427298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,31,0.009317333499590555
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,31,0.009205333267649015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,63,0.009466666728258133
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,63,0.008943999807039896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,127,0.009370666618148485
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,127,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,255,0.00901333304742972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,255,0.008986666798591614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,511,0.010938666760921478
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,511,0.01101333275437355
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,1023,0.01128000020980835
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,1023,0.010954666882753372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,2047,0.010879999647537867
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,2047,0.010949333508809408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,4095,0.011136000355084738
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,4095,0.01099733387430509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,8191,0.01621333385507266
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,8191,0.016117333124081295
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,16383,0.019178666174411774
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,16383,0.019274666905403137
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,32767,0.021173333128293354
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,32767,0.020517333100239437
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,127,0.009402666861812273
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,1,0.009312000125646591
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,1,0.00901333304742972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,3,0.009610666582981745
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,3,0.008943999807039896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,7,0.009082666908701261
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,511,0.009269333134094873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,7,0.009733333562811216
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,15,0.009066666786869368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,31,0.008922666932145754
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,31,0.008976000050703684
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,63,0.009103999783595404
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,63,0.009114666531483332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,127,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,127,0.009002666920423508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,255,0.009066666786869368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,255,0.009279999881982803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,511,0.011285333583752314
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,511,0.008976000050703684
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,1023,0.011045332998037338
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,2047,0.010501333822806677
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,2047,0.01108266661564509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,4095,0.013050666699806849
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,3,0.009173333023985228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,4095,0.013050666699806849
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,8191,0.015311999867359797
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,8191,0.01526933287580808
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,16383,0.01718933383623759
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,16383,0.017077332983414333
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,32767,0.03363733241955439
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,1,0.008965333302815756
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,1,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,3,0.009226666763424873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,3,0.009359999870260557
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,7,0.009322666873534521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,7,0.009045333291093508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,15,0.009152000149091085
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,31,0.00916800027092298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,31,0.00926399976015091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,63,0.008992000172535578
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,63,0.009269333134094873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,127,0.009530666594703993
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,127,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,255,0.00985599992175897
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,511,0.010938666760921478
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,1023,0.01089599976936976
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,1023,0.010714666297038397
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,15,0.009178666397929192
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,2047,0.01293333371480306
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,4095,0.015168000012636185
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,4095,0.015082667271296183
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,8191,0.017184000462293625
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,8191,0.01509333277742068
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,32767,0.01937599976857503
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,16383,0.03141866624355316
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,16383,0.017818666994571686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,32767,0.03342933456103007
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,32767,0.05003733436266581
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,1,0.011002667248249054
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,15,0.008997333546479544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,1,0.010992000500361124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,3,0.011152000476916632
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,3,0.011530666301647821
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,7,0.011077333241701126
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,7,0.010911999891201654
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,15,0.010944000134865442
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,255,0.009183999771873156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,15,0.010853332777818045
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,511,0.010805333654085795
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,31,0.01108266661564509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,31,0.010965333630641302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,63,0.010960000256697336
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,127,0.01091733326514562
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,127,0.01090666651725769
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,2047,0.013072000195582708
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,255,0.011066666493813196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,1023,0.009866666669646898
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,255,0.011231999844312668
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,511,0.011754666765530905
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,511,0.011312000453472137
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,1023,0.01321600005030632
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,1023,0.01328533391157786
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,4095,0.04879466692606608
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,4095,0.032144000132878624
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,2047,0.029477333029111225
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,1,0.01116266722480456
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,1,0.011050666371981302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,3,0.011525332927703857
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,3,0.011306667079528173
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,7,0.010960000256697336
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,7,0.01191466674208641
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,15,0.011402666568756104
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,15,0.01110400011142095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,31,0.01166933278242747
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,31,0.011168000598748526
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,63,0.011242666592200598
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,63,0.011029332876205444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,127,0.011087999989589056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,127,0.011274666835864386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,63,0.011141333729028702
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,255,0.010981333752473196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,255,0.011018666128317514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,511,0.015008000036080679
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,511,0.013151999562978745
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,1023,0.015418666104475657
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,1023,0.029968000948429108
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,2047,0.03148799886306127
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,4095,0.08519466718037923
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,4095,0.05120000243186951
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,1,0.013973332941532135
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,3,0.013056000073750814
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,2047,0.017312000195185345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,7,0.01339200014869372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,7,0.014912000546852747
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,1,0.013338666409254074
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,3,0.015301333119471868
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,15,0.015109332899252573
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,15,0.01303999995191892
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,31,0.013376000026861826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,63,0.015119999647140503
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,63,0.013248000293970108
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,127,0.015024000157912573
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,127,0.013514666507641474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,31,0.015274666249752045
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,255,0.015114666273196539
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,255,0.013365333278973898
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,1023,0.04842666784922282
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,1023,0.031583999594052635
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,511,0.03081600119670232
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,2047,0.08594666918118794
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,1,0.009093333035707474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,2047,0.05013333261013031
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,3,0.009226666763424873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,2047,0.0481333335240682
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,4095,0.08636800448099773
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,4095,0.15873066584269205
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,7,0.009216000015536943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,7,0.009109333157539368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,1,0.009066666786869368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,15,0.008869333192706108
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,15,0.009279999881982803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,31,0.008992000172535578
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,31,0.008997333546479544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,63,0.009216000015536943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,63,0.010757333288590113
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,127,0.009194666519761086
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,127,0.010981333752473196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,255,0.009114666531483332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,255,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,511,0.010832000523805618
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,1023,0.010869332899649939
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,1023,0.01090666651725769
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,2047,0.01071999967098236
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,2047,0.011018666128317514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,4095,0.015077333897352219
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,4095,0.014970666418472925
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,8191,0.017136000096797943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,8191,0.015728000551462173
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,16383,0.01931200052301089
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,16383,0.01718933383623759
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,32767,0.03475199888149897
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,32767,0.021520001192887623
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,1,0.009285333255926767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,1,0.008992000172535578
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,3,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,3,0.00915733352303505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,7,0.00897066667675972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,7,0.009242666885256767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,15,0.009312000125646591
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,15,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,31,0.009226666763424873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,31,0.008938666433095932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,63,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,127,0.009119999905427298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,127,0.009194666519761086
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,255,0.00895999992887179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,255,0.009514666472872099
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,511,0.010992000500361124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,511,0.01718933383623759
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,511,0.010970667004585266
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,1023,0.010933333386977514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,1023,0.010650667051474253
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,2047,0.012981332838535309
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,3,0.010687999427318573
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,2047,0.012970666090647379
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,4095,0.01368533323208491
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,4095,0.0136266661187013
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,8191,0.017082666357358296
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,8191,0.015157333264748255
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,16383,0.018112000077962875
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,16383,0.03148799886306127
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,32767,0.05009600023428599
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,32767,0.03333866596221924
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,1,0.00922133338948091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,511,0.011098666737476984
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,3,0.00901333304742972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,7,0.008933333059151968
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,7,0.009194666519761086
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,15,0.00915733352303505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,15,0.009066666786869368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,31,0.008879999940594038
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,31,0.009285333255926767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,63,0.009557333464423815
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,63,0.008997333546479544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,127,0.009173333023985228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,255,0.009888000165422758
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,255,0.009039999917149544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,63,0.008885333314538002
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,511,0.01097600037852923
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,511,0.010954666882753372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,1023,0.011120000233252844
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,2047,0.01232533281048139
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,2047,0.01302933320403099
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,4095,0.015333333363135656
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,4095,0.01314666618903478
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,8191,0.02941333254178365
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,8191,0.01695999999841054
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,16383,0.031514666974544525
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,16383,0.0484799991051356
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,32767,0.04961066444714864
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,1,0.00914666677514712
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,3,0.009162666896979014
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,1,0.013114667187134424
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,1,0.013088000317414602
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,3,0.013248000293970108
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,3,0.013072000195582708
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,7,0.013178666432698568
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,7,0.013023999830087027
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,127,0.009253333633144697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,15,0.013258667041858038
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,15,0.01309866706530253
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,1023,0.01099733387430509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,31,0.013088000317414602
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,31,0.01309866706530253
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,63,0.013082666943470636
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,63,0.012965332716703415
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,127,0.012901333471139273
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,127,0.012709333250919977
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,255,0.013077333569526672
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,255,0.012831999609867731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,511,0.015290666371583939
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,32767,0.08462400237719218
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,511,0.015029333531856537
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,1023,0.031498665610949196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,1023,0.017680000513792038
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,1,0.015194666882356008
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,3,0.014975999792416891
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,2047,0.03363200028737386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,1,0.0136266661187013
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,15,0.01523200049996376
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,7,0.01310933381319046
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,7,0.015082667271296183
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,15,0.013408000270525614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,31,0.013258667041858038
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,31,0.015130666395028433
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,63,0.015237333873907724
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,127,0.015018666783968607
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,127,0.013301332791646322
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,255,0.015226667126019796
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,255,0.013722666849692663
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,511,0.03150933235883713
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,511,0.017082666357358296
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,1023,0.04948266843954722
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,2047,0.04993600149949392
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,2047,0.08636800448099773
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,3,0.013280000537633896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,1023,0.03209600100914637
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,2047,0.05175999800364176
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,1,0.019130667050679524
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,3,0.01714133347074191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,7,0.019167999426523846
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,63,0.013424000392357508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,15,0.019248000035683315
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,1,0.017210666090250015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,15,0.017263999829689663
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,3,0.019167999426523846
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,31,0.01926933353145917
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,7,0.01729600007335345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,31,0.017450666675964992
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,63,0.01720000058412552
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,63,0.01926400015751521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,127,0.01934933289885521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,127,0.017488000293572743
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,255,0.030826665461063385
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,255,0.017632000148296356
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,1023,0.08496000369389851
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,511,0.048453330993652344
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,1023,0.05131733417510986
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,511,0.0313226655125618
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,2047,0.1567626694838206
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,2047,0.08733333150545756
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,1,0.015178666760524115
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,1,0.017018667111794155
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,3,0.015040000279744467
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,7,0.015210667004187902
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,15,0.016437333077192307
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,3,0.015509333461523056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,7,0.01709866647919019
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,15,0.015109332899252573
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,31,0.015135999768972397
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,31,0.017344000438849132
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,63,0.01570133368174235
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,63,0.015024000157912573
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,127,0.015850666910409927
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,127,0.015114666273196539
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,255,0.016336000214020412
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,255,0.015247999380032221
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,511,0.03186133255561193
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,511,0.01939733326435089
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,1,0.019354666272799175
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,3,0.01922133316596349
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,3,0.017535999417304993
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,7,0.019434666881958645
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,7,0.017610666652520496
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,15,0.019391999890406925
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,31,0.019333332777023315
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,31,0.017407999684413273
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,15,0.017605333278576534
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,63,0.019306667149066925
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,63,0.017583999782800674
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,127,0.01930133377512296
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,127,0.018112000077962875
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,1,0.01746133342385292
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,1,0.029535998900731403
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,255,0.017407999684413273
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,255,0.0321066677570343
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,511,0.032405334214369454
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,511,0.05002133548259735
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,1,0.025519999365011852
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,3,0.02937600016593933
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,3,0.025631998976071674
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,7,0.0303413321574529
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,7,0.02569599946339925
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,31,0.02716800073782603
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,15,0.02959999938805898
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,63,0.026191999514897663
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,15,0.02534399926662445
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,31,0.02939733366171519
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,127,0.033770665526390076
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,127,0.026757332185904186
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,255,0.05028266708056132
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,255,0.033615998923778534
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,63,0.029631999631722767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,1,0.02349333216746648
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,511,0.08608532945315044
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,1,0.020986666282018025
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,511,0.05027733246485392
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,3,0.02333866556485494
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,3,0.01966933285196622
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,7,0.02348800003528595
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,7,0.01923199991385142
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,31,0.02311466634273529
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,15,0.023178666830062866
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,63,0.0232640008131663
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,31,0.01932266727089882
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,15,0.02051199972629547
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,127,0.02342933416366577
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,127,0.019285333653291065
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,255,0.03149333347876867
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,1,0.03145600110292435
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,255,0.020010666300853092
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,1,0.027701333165168762
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,3,0.03146133323510488
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,3,0.027456000447273254
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,63,0.019285333653291065
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,7,0.03148799886306127
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,15,0.03152533372243246
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,15,0.027818667391935985
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,31,0.03153600047032038
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,31,0.02848000079393387
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,63,0.02758399893840154
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,63,0.03163733333349228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,127,0.035461333890755974
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,127,0.029343999922275543
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,7,0.028037334481875103
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,255,0.05239999790986379
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,255,0.037392000357309975
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,1,0.04455466568470001
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,1,0.054197331269582115
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,3,0.05301333467165629
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,3,0.04484800000985464
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,7,0.05409066875775655
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,15,0.04422399898370107
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,7,0.0440533310174942
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,15,0.053898667295773826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,31,0.05409599840641022
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,31,0.04378666480382284
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,1,0.009343999748428663
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,63,0.05392000079154968
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,1,0.009322666873534521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,63,0.045125335454940796
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,3,0.00901333304742972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,3,0.008901333436369896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,7,0.009375999992092451
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,15,0.011007999380429586
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,31,0.008976000050703684
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,127,0.057087997595469155
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,31,0.01102399950226148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,63,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,7,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,127,0.00926399976015091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,15,0.009002666920423508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,127,0.009029333169261614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,255,0.008879999940594038
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,255,0.010938666760921478
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,511,0.01099733387430509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,511,0.011045332998037338
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,1023,0.01102399950226148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,1023,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,2047,0.013301332791646322
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,2047,0.013210666676362356
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,4095,0.015141333142916361
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,63,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,4095,0.015141333142916361
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,8191,0.017514667163292568
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,8191,0.017221332838137943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,16383,0.033402666449546814
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,127,0.04791999856630961
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,32767,0.05246399839719137
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,32767,0.03578133384386698
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,1,0.009743999689817429
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,1,0.009610666582981745
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,3,0.009173333023985228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,7,0.009253333633144697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,15,0.008954666554927826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,15,0.009328000247478485
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,31,0.009285333255926767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,3,0.009541333342591921
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,31,0.008943999807039896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,63,0.00892800030608972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,63,0.009269333134094873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,127,0.009759999811649323
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,127,0.008965333302815756
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,255,0.05724266668160757
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,255,0.00895999992887179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,255,0.009205333267649015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,511,0.011157333850860596
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,511,0.01099733387430509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,255,0.0892639954884847
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,1023,0.01098666712641716
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,2047,0.013130666067202887
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,2047,0.01098666712641716
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,4095,0.015072000523408255
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,8191,0.030389333764712017
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,8191,0.015520000209410986
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,16383,0.04788800080617269
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,32767,0.08469333251317342
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,1,0.009178666397929192
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,1023,0.011120000233252844
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,1,0.011029332876205444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,3,0.009178666397929192
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,3,0.009306666751702627
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,16383,0.020053333292404812
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,7,0.009717333440979322
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,4095,0.013397333522637686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,7,0.009312000125646591
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,15,0.009205333267649015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,15,0.009141333401203156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,31,0.00922133338948091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,31,0.00927466650803884
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,63,0.009136000027259191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,63,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,127,0.009194666519761086
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,127,0.009343999748428663
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,255,0.009194666519761086
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,511,0.010960000256697336
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,511,0.010847999403874079
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,1023,0.011237333218256632
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,1023,0.011343999455372492
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,2047,0.01505600040157636
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,4095,0.02926933268706004
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,4095,0.015882667154073715
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,8191,0.04819199939568838
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,16383,0.0313226655125618
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,8191,0.03143466760714849
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,32767,0.050026665131251015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,16383,0.04851733148097992
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,16383,0.0844640036424001
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,32767,0.15567466616630554
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,32767,0.08518933256467183
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,7,0.009098666409651438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,255,0.009088000282645226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,2047,0.013056000073750814
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,1,0.03159466634194056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,1,0.037647999823093414
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,3,0.039605334401130676
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,3,0.03205333401759466
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,7,0.03771200031042099
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,7,0.03226666649182638
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,15,0.03751466671625773
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,15,0.03176533430814743
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,31,0.03842133283615112
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,31,0.03196800003449122
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,63,0.03756266583998998
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,63,0.03180799881617228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,127,0.039674667020638786
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,127,0.03346133232116699
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,1,0.05614933371543884
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,1,0.049866666396458946
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,3,0.057861333092053734
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,3,0.049456000328063965
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,7,0.04982399940490723
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,7,0.05825066566467285
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,15,0.0498933345079422
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,31,0.05723733206590017
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,31,0.04994133114814758
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,63,0.057317331433296204
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,63,0.049882665276527405
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,15,0.05750933289527893
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,127,0.06211199859778086
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,127,0.05413866539796194
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,1,0.09520000219345093
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,1,0.08382933338483174
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,3,0.09689066807428996
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,3,0.08471999565760295
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,7,0.09715200463930766
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,7,0.08363200227419536
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,31,0.09687466422716777
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,15,0.09528000156084697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,15,0.08495466907819112
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,31,0.08455466230710347
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,63,0.09717866778373718
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,63,0.08462400237719218
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,127,0.10309867064158122
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,127,0.08658666412035625
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,1,0.056405335664749146
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,float16,1,0.0685280015071233
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,float16,3,0.06854933500289917
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,3,0.05695466697216034
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,float16,7,0.06834666430950165
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,7,0.05604266623655955
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,float16,15,0.07032000025113423
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,15,0.0562666654586792
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,float16,31,0.0669653316338857
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,31,0.0581279993057251
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,float16,63,0.06720533470312755
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,63,0.05622933308283488
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,1,0.09353066484133403
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,float16,3,0.10347732901573181
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,float16,1,0.10313066840171814
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,3,0.09470933675765991
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,float16,7,0.10285866260528564
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,7,0.09499733646710713
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,15,0.09499200185139973
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,float16,15,0.10321600238482158
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,float16,31,0.1034399966398875
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,31,0.09512000282605489
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,63,0.09508267045021057
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,float16,1,0.1771413286526998
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,1,0.15627200404802957
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,float16,63,0.10296533505121867
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,3,0.15689067045847574
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,float16,3,0.17775466044743857
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,float16,7,0.1772586703300476
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,7,0.15637866655985513
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,1,0.009269333134094873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,1,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,3,0.00926399976015091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,7,0.009029333169261614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,7,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,15,0.00895999992887179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,15,0.00895999992887179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,31,0.008869333192706108
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,31,0.00927466650803884
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,63,0.009493333597977957
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,63,0.010992000500361124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,float16,15,0.17841066916783652
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,127,0.009258666386206945
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,15,0.15458133816719055
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,127,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,3,0.009061333412925402
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,255,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,511,0.010911999891201654
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,255,0.010928000013033548
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,511,0.011130666981140772
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,1023,0.01121066634853681
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,float16,31,0.17950934171676636
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,1023,0.011365332951148352
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,2047,0.012944000462690989
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,2047,0.013061333447694778
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,4095,0.015119999647140503
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,8191,0.03142933299144109
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,8191,0.017786666750907898
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,16383,0.04965866605440775
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,31,0.1557866632938385
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,16383,0.03275199979543686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,1,0.009226666763424873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,1,0.009045333291093508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,3,0.009093333035707474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,7,0.00916800027092298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,7,0.009984000275532404
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,15,0.009173333023985228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,3,0.009226666763424873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,15,0.00914666677514712
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,31,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,31,0.009162666896979014
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,63,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,63,0.010885333021481832
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,127,0.009045333291093508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,127,0.009663999701539675
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,255,0.009285333255926767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,255,0.010698666175206503
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,511,0.011007999380429586
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,511,0.010960000256697336
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,1023,0.011391999820868174
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,1023,0.012138667205969492
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,2047,0.013327999661366144
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,2047,0.01331199953953425
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,4095,0.030069333811601002
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,4095,0.01718933383623759
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,8191,0.031301334500312805
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,8191,0.04818666477998098
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,16383,0.0497920016447703
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,16383,0.08461866776148479
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,1,0.010138666878143946
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,1,0.010992000500361124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,3,0.011071999867757162
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,63,0.15556800365447998
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,7,0.010469333579142889
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,15,0.01089599976936976
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,7,0.008949333180983862
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,15,0.011018666128317514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,31,0.01073066641887029
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,31,0.01128000020980835
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,63,0.009493333597977957
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,127,0.010965333630641302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,127,0.010373333469033241
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,255,0.0106133334338665
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,255,0.011007999380429586
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,511,0.011253333340088526
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,511,0.011141333729028702
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,1023,0.013034666577974955
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,1023,0.013013333082199097
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,2047,0.015253332753976187
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,4095,0.030000001192092896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,4095,0.04781866570313772
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,8191,0.04841599861780802
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,8191,0.0837653378645579
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,16383,0.15619200468063354
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,1,0.011141333729028702
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,16383,0.08461333314577739
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,1,0.011018666128317514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,float16,63,0.17899733781814575
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,3,0.011183999478816986
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,4095,0.014959999670584997
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,3,0.01091733326514562
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,7,0.009018666421373686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,7,0.011109333485364914
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,15,0.009594666461149851
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,31,0.010575999816258749
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,31,0.011029332876205444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,63,0.011087999989589056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,63,0.010853332777818045
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,127,0.010970667004585266
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,127,0.010954666882753372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,255,0.011114666859308878
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,255,0.010928000013033548
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,511,0.011274666835864386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,511,0.011264000087976456
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,15,0.012442667037248611
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,1023,0.013242666920026144
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,1023,0.012991999586423239
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,2047,0.015066667149464289
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,4095,0.031514666974544525
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,63,0.010954666882753372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,4095,0.017269333203633625
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,8191,0.050442665815353394
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,1,0.011109333485364914
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,1,0.010879999647537867
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,3,0.011045332998037338
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,7,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,3,0.010431999961535135
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,7,0.010938666760921478
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,15,0.010805333654085795
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,15,0.010703999549150467
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,3,0.010645333677530289
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,31,0.010858666151762009
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,31,0.010389333590865135
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,63,0.011045332998037338
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,127,0.010928000013033548
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,127,0.01099733387430509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,255,0.011055999745925268
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,255,0.01080000028014183
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,511,0.011205332974592844
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,1023,0.013088000317414602
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,1023,0.013066666821638743
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,2047,0.02940266579389572
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,2047,0.02959999938805898
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,2047,0.015109332899252573
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,8191,0.03180799881617228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,4095,0.047872001926104225
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,4095,0.03065599997838338
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,8191,0.04910933474699656
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,1,0.011109333485364914
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,1,0.011045332998037338
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,3,0.01109333336353302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,7,0.011413333316644033
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,63,0.011061333119869232
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,15,0.011381333072980246
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,15,0.011333333949247995
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,31,0.011157333850860596
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,31,0.011120000233252844
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,511,0.01099733387430509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,63,0.01110400011142095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,63,0.011125333607196808
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,127,0.011141333729028702
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,127,0.011274666835864386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,255,0.011296000331640244
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,255,0.011130666981140772
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,2047,0.014965333044528961
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,511,0.013829333086808523
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,511,0.013130666067202887
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,1023,0.015295999745527903
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,2047,0.04789333542188009
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,8191,0.08426133791605632
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,2047,0.031680000325044
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,4095,0.08473599950472514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,4095,0.05004266897837321
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,1,0.00915733352303505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,1,0.009301333377758661
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,3,0.01102399950226148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,3,0.009045333291093508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,3,0.009114666531483332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,8191,0.1570186714331309
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,8191,0.08685866991678874
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,7,0.009088000282645226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,7,0.009872000043590864
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,15,0.009141333401203156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,31,0.009103999783595404
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,63,0.009045333291093508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,63,0.010954666882753372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,127,0.00901333304742972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,127,0.00903466654320558
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,255,0.009354666496316591
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,255,0.010954666882753372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,511,0.010890666395425797
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,511,0.010992000500361124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,1023,0.010911999891201654
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,1023,0.011077333241701126
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,2047,0.011157333850860596
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,2047,0.01098666712641716
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,4095,0.01128000020980835
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,8191,0.015279999623696009
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,8191,0.017045332739750545
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,16383,0.021274665991465252
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,16383,0.021359999974568684
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,32767,0.023306667804718018
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,32767,0.023317334552605946
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,1,0.00890666681031386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,1,0.00903466654320558
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,3,0.009130666653315226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,3,0.008954666554927826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,7,0.009098666409651438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,7,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,15,0.008943999807039896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,1023,0.030053332448005676
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,15,0.009098666409651438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,31,0.009018666421373686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,31,0.008943999807039896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,63,0.009018666421373686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,63,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,127,0.00895999992887179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,127,0.008976000050703684
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,255,0.009039999917149544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,255,0.009029333169261614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,511,0.010874666273593903
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,511,0.011071999867757162
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,1023,0.011109333485364914
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,7,0.010954666882753372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,15,0.00961599995692571
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,2047,0.010928000013033548
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,4095,0.011071999867757162
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,31,0.009194666519761086
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,4095,0.010981333752473196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,8191,0.015173333386580149
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,8191,0.017263999829689663
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,16383,0.017450666675964992
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,16383,0.0173333336909612
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,32767,0.020608000457286835
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,32767,0.019109333554903667
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,1,0.009557333464423815
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,4095,0.011296000331640244
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,1,0.009088000282645226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,3,0.008997333546479544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,7,0.009722666814923286
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,7,0.009925333162148794
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,15,0.009088000282645226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,15,0.008976000050703684
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,31,0.009125333279371262
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,31,0.009695999945203463
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,63,0.009141333401203156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,127,0.011055999745925268
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,127,0.009189333145817121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,255,0.009888000165422758
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,255,0.010384000216921171
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,511,0.010816000401973724
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,511,0.011253333340088526
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,1023,0.011077333241701126
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,1023,0.010069333637754122
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,2047,0.01110400011142095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,2047,0.010794666906197866
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,4095,0.013189333180586496
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,1023,0.010869332899649939
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,4095,0.013077333569526672
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,2047,0.009573333586255709
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,8191,0.015168000012636185
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,8191,0.015184000134468079
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,16383,0.01695999999841054
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,16383,0.01736533393462499
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,1,0.010458666831254959
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,32767,0.0191040001809597
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,32767,0.033413333197434746
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,1,0.009354666496316591
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,3,0.009045333291093508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,3,0.010773333410422007
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,7,0.009253333633144697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,7,0.00898133342464765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,15,0.009077333534757296
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,31,0.009039999917149544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,31,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,63,0.008890666688481966
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,63,0.009290666629870733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,127,0.008938666433095932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,127,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,255,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,255,0.008986666798591614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,511,0.01090666651725769
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,511,0.010816000401973724
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,1023,0.010837333897749582
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,2047,0.010938666760921478
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,2047,0.011114666859308878
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,4095,0.010960000256697336
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,4095,0.01110400011142095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,63,0.00956266683836778
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,8191,0.017290666699409485
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,16383,0.01899733394384384
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,32767,0.021370666722456615
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,32767,0.0210506667693456
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,1,0.009642666826645533
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,1,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,3,0.00915733352303505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,3,0.009258666386206945
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,7,0.009056000038981438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,7,0.009130666653315226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,15,0.009242666885256767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,15,0.008986666798591614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,31,0.009103999783595404
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,31,0.009253333633144697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,63,0.009136000027259191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,63,0.009242666885256767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,127,0.008938666433095932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,127,0.00979200005531311
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,255,0.009077333534757296
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,255,0.00927466650803884
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,511,0.010842667271693548
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,511,0.01090666651725769
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,15,0.009039999917149544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,1023,0.010853332777818045
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,1023,0.01101333275437355
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,3,0.00897066667675972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,2047,0.01089599976936976
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,2047,0.011114666859308878
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,4095,0.013514666507641474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,4095,0.013013333082199097
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,8191,0.015119999647140503
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,8191,0.015285332997639975
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,16383,0.01728533332546552
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,16383,0.017301333447297413
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,32767,0.034101332227389015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,1023,0.011045332998037338
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,32767,0.01932799940307935
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,1,0.009114666531483332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,1,0.009178666397929192
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,8191,0.017077332983414333
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,3,0.009061333412925402
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,3,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,7,0.008938666433095932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,7,0.009114666531483332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,16383,0.019109333554903667
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,15,0.009312000125646591
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,15,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,31,0.009877333417534828
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,63,0.009269333134094873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,63,0.009301333377758661
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,127,0.00895999992887179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,127,0.009301333377758661
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,255,0.009125333279371262
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,255,0.009285333255926767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,511,0.01109333336353302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,1023,0.010885333021481832
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,1023,0.009472000102202097
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,2047,0.0124746672809124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,4095,0.01333333303531011
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,4095,0.015194666882356008
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,8191,0.01714133347074191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,8191,0.015194666882356008
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,16383,0.01791999985774358
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,16383,0.0322826678554217
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,32767,0.03368533402681351
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,32767,0.04997866849104563
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,1,0.011157333850860596
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,1,0.01110400011142095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,3,0.010847999403874079
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,3,0.010944000134865442
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,7,0.010954666882753372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,15,0.010863999525705973
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,15,0.01091733326514562
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,31,0.01098666712641716
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,31,0.011066666493813196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,63,0.010954666882753372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,31,0.009296000003814697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,63,0.010911999891201654
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,127,0.011141333729028702
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,511,0.01098666712641716
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,255,0.011077333241701126
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,255,0.011002667248249054
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,2047,0.013072000195582708
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,511,0.011541333049535751
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,511,0.011306667079528173
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,1023,0.014938666174809137
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,2047,0.02956266701221466
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,4095,0.04868799944718679
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,2047,0.01741333305835724
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,4095,0.031557333966096245
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,1,0.011850666254758835
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,3,0.011541333049535751
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,7,0.011253333340088526
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,7,0.01146666705608368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,1,0.011152000476916632
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,7,0.011071999867757162
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,3,0.01129066695769628
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,15,0.011535999675591787
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,15,0.011018666128317514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,31,0.011802667131026586
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,31,0.011007999380429586
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,63,0.011962667107582092
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,63,0.011226666470368704
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,127,0.011482667177915573
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,127,0.010890666395425797
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,255,0.010954666882753372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,255,0.011055999745925268
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,511,0.01504533365368843
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,511,0.013264000415802002
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,2047,0.0315733328461647
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,1023,0.01569066693385442
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,1023,0.013178666432698568
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,4095,0.08474133412043254
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,4095,0.05004266897837321
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,1,0.015200000256299973
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,1,0.01313599944114685
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,3,0.013290667285521826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,3,0.015040000279744467
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,7,0.014282666146755219
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,7,0.013056000073750814
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,15,0.015098666151364645
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,31,0.015061333775520325
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,63,0.015061333775520325
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,63,0.01309866706530253
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,127,0.015040000279744467
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,127,0.011071999867757162
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,127,0.013130666067202887
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,255,0.015098666151364645
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,255,0.013397333522637686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,2047,0.0481333335240682
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,1023,0.02992533395687739
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,511,0.0295413335164388
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,1023,0.0490880012512207
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,1023,0.03155199935038885
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,2047,0.08611733714739482
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,1,0.00914666677514712
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,2047,0.050016000866889954
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,1,0.009258666386206945
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,4095,0.1590986649195353
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,3,0.009173333023985228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,3,0.009301333377758661
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,7,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,31,0.01322666679819425
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,15,0.009178666397929192
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,15,0.010890666395425797
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,31,0.009189333145817121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,63,0.009226666763424873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,63,0.00980266680320104
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,127,0.009301333377758661
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,127,0.009472000102202097
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,255,0.009093333035707474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,255,0.009370666618148485
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,511,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,511,0.01102399950226148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,1023,0.011055999745925268
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,1023,0.010885333021481832
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,511,0.017045332739750545
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,2047,0.01108266661564509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,2047,0.010933333386977514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,4095,0.015087999403476715
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,8191,0.01695999999841054
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,8191,0.017082666357358296
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,16383,0.019189332922299702
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,16383,0.0179626668492953
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,32767,0.03510933369398117
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,32767,0.021317332983016968
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,1,0.00922133338948091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,4095,0.0883786678314209
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,1,0.009269333134094873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,3,0.009226666763424873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,3,0.009061333412925402
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,7,0.009119999905427298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,7,0.00916800027092298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,15,0.009173333023985228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,7,0.009109333157539368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,15,0.009189333145817121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,31,0.009130666653315226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,31,0.009328000247478485
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,63,0.00891733355820179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,127,0.009136000027259191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,127,0.009183999771873156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,255,0.009008000294367472
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,255,0.009056000038981438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,511,0.010821333775917688
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,511,0.011077333241701126
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,1023,0.00897066667675972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,1023,0.01071999967098236
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,2047,0.01166933278242747
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,2047,0.012901333471139273
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,4095,0.014533333480358124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,4095,0.01302933320403099
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,8191,0.015344000111023584
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,8191,0.01729600007335345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,16383,0.03182933231194814
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,16383,0.019296000401178997
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,32767,0.03359466542800268
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,1,0.00915733352303505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,1,0.008997333546479544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,4095,0.015247999380032221
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,3,0.00914666677514712
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,3,0.010069333637754122
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,7,0.008986666798591614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,15,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,15,0.008752000207702318
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,31,0.010725333044926325
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,63,0.009103999783595404
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,15,0.013290667285521826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,63,0.009136000027259191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,127,0.009002666920423508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,63,0.009205333267649015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,127,0.00898133342464765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,255,0.009050666665037474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,255,0.009493333597977957
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,511,0.011141333729028702
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,1023,0.011061333119869232
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,1023,0.010970667004585266
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,2047,0.013082666943470636
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,2047,0.01116266722480456
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,4095,0.015205333630243937
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,4095,0.01309866706530253
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,8191,0.02978666623433431
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,8191,0.015066667149464289
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,32767,0.050426666935284935
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,16383,0.04822933177153269
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,16383,0.031471999982992806
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,32767,0.08360532919565837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,32767,0.04986133178075155
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,7,0.009365333244204521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,1,0.013050666699806849
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,1,0.012890666723251343
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,31,0.009183999771873156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,3,0.013189333180586496
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,3,0.012938667088747025
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,7,0.012815999488035837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,7,0.013082666943470636
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,31,0.010319999729593595
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,15,0.013125333935022354
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,15,0.01302933320403099
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,31,0.013045333325862885
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,31,0.013157332936922709
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,511,0.010965333630641302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,63,0.013023999830087027
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,63,0.013088000317414602
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,127,0.012847999731699625
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,127,0.013066666821638743
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,255,0.013077333569526672
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,255,0.013194666554530462
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,511,0.015247999380032221
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,511,0.015168000012636185
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,1023,0.03146133323510488
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,1023,0.01798933371901512
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,1,0.01498666654030482
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,1,0.013295999417702356
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,2047,0.04985066751639048
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,2047,0.03340800106525421
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,3,0.015274666249752045
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,3,0.013376000026861826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,7,0.013338666409254074
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,15,0.015002666662136713
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,7,0.015189333508412043
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,31,0.01328533391157786
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,63,0.015077333897352219
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,63,0.013248000293970108
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,15,0.014117332796255747
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,127,0.015018666783968607
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,127,0.013487999637921652
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,31,0.015189333508412043
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,255,0.01515199989080429
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,255,0.013199999928474426
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,2047,0.08699733018875122
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,511,0.03062933435042699
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,511,0.017184000462293625
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,2047,0.05161599814891815
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,1,0.017242666333913803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,1,0.019226666539907455
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,1023,0.04980266590913137
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,1023,0.03213333338499069
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,3,0.01923199991385142
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,7,0.01732800031701724
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,15,0.017055999487638474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,3,0.017263999829689663
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,7,0.01923199991385142
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,63,0.01926400015751521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,63,0.017386666188637417
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,127,0.019173332800467808
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,127,0.017258666455745697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,31,0.017349333812793095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,255,0.030432000756263733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,255,0.01711999997496605
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,1023,0.05068266888459524
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,1023,0.08563733100891113
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,511,0.03141333411137263
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,511,0.048528000712394714
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,2047,0.15705600380897522
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,15,0.01926400015751521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,2047,0.08672533432642619
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,1,0.01523200049996376
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,1,0.01524266724785169
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,7,0.01595199977358182
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,7,0.015157333264748255
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,15,0.016458666572968166
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,3,0.015098666151364645
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,31,0.019130667050679524
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,3,0.016597333053747814
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,15,0.015317333241303762
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,31,0.017312000195185345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,31,0.015018666783968607
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,63,0.016106666376193363
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,63,0.015290666371583939
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,127,0.015087999403476715
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,127,0.014965333044528961
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,255,0.016415999581416447
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,255,0.015135999768972397
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,511,0.03160533308982849
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,1,0.019285333653291065
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,511,0.01945066700379054
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,3,0.01921066641807556
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,3,0.017418666432301205
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,7,0.019317333896954853
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,7,0.017279999951521557
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,15,0.017498667041460674
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,15,0.019402666638294857
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,31,0.018394666413466137
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,31,0.01921066641807556
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,63,0.019178666174411774
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,63,0.017360000560681026
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,127,0.019205333044131596
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,127,0.017152000218629837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,1,0.017418666432301205
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,255,0.017792000124851864
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,255,0.03084266682465871
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,1,0.029440000653266907
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,511,0.032458665470282234
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,1,0.026842666169007618
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,3,0.029445332785447437
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,3,0.025722667574882507
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,7,0.029552000264326733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,15,0.029509333272775013
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,7,0.02550400048494339
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,31,0.030320001145203907
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,15,0.025765334566434223
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,31,0.025557334224383037
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,63,0.02958933264017105
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,63,0.025450666745503742
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,511,0.04902400076389313
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,127,0.0335413341720899
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,127,0.026202666262785595
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,255,0.049509331583976746
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,255,0.03254933406909307
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,1,0.019205333044131596
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,1,0.023200000325838726
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,511,0.08653333783149719
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,511,0.05085866649945577
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,3,0.023599999646345775
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,3,0.019343999524911244
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,7,0.0232640008131663
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,7,0.020202666521072388
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,15,0.023402666052182514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,15,0.0194560003777345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,63,0.02351466566324234
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,63,0.019610666980346043
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,127,0.023200000325838726
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,31,0.023317334552605946
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,31,0.019839999576409657
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,127,0.019461333751678467
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,255,0.019440000255902607
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,1,0.02887466549873352
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,3,0.03143999973932902
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,3,0.027701333165168762
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,7,0.027690666417280834
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,7,0.03084266682465871
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,15,0.03149333347876867
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,15,0.02755733331044515
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,255,0.032399999598662056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,31,0.031386665999889374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,1,0.03182933231194814
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,31,0.027386667827765148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,63,0.03146133323510488
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,63,0.027744000156720478
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,127,0.035461333890755974
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,127,0.029509333272775013
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,255,0.05231466889381409
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,1,0.043791999419530235
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,1,0.054042667150497437
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,255,0.036506667733192444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,3,0.051962668697039284
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,3,0.04385599990685781
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,7,0.053930665055910744
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,7,0.04561600089073181
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,15,0.04373333354791006
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,31,0.05392000079154968
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,15,0.053642665346463524
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,63,0.05329066514968872
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,63,0.04404800136884054
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,31,0.04588800172011057
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,1,0.009328000247478485
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,7,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,7,0.00903466654320558
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,3,0.01181866725285848
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,15,0.009306666751702627
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,127,0.05693866809209188
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,1,0.009226666763424873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,15,0.009130666653315226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,3,0.009370666618148485
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,31,0.00898133342464765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,31,0.008992000172535578
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,63,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,127,0.009322666873534521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,127,0.00915733352303505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,255,0.00877333308259646
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,255,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,511,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,511,0.011029332876205444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,1023,0.010922666639089584
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,1023,0.010879999647537867
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,2047,0.013125333935022354
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,2047,0.013397333522637686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,4095,0.015178666760524115
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,4095,0.01505600040157636
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,8191,0.01724799970785777
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,16383,0.033413333197434746
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,16383,0.020954666038354237
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,32767,0.05188799897829691
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,32767,0.03477333237727483
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,1,0.008885333314538002
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,1,0.010480000327030817
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,3,0.008912000184257826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,3,0.009189333145817121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,255,0.08875733613967896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,7,0.009008000294367472
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,7,0.009008000294367472
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,15,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,15,0.009077333534757296
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,255,0.05624000231424967
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,31,0.009216000015536943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,63,0.009253333633144697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,63,0.009189333145817121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,127,0.009018666421373686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,127,0.010581333190202713
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,255,0.009178666397929192
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,31,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,511,0.010922666639089584
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,511,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,1023,0.01102399950226148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,1023,0.010853332777818045
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,2047,0.012901333471139273
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,2047,0.01139733319481214
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,63,0.00898133342464765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,4095,0.01509333277742068
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,4095,0.013258667041858038
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,8191,0.030031998952229817
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,127,0.04809600114822388
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,8191,0.017125333348910015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,16383,0.04866133133570353
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,32767,0.08468266328175862
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,32767,0.04987200101216634
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,1,0.011061333119869232
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,3,0.009242666885256767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,3,0.010005333150426546
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,7,0.009226666763424873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,1,0.009050666665037474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,7,0.008885333314538002
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,15,0.009088000282645226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,15,0.009381333366036415
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,31,0.009109333157539368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,31,0.009290666629870733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,63,0.010741333166758219
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,63,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,127,0.00922133338948091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,127,0.010464000205198923
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,255,0.009541333342591921
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,255,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,255,0.009269333134094873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,511,0.011050666371981302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,511,0.01089599976936976
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,1023,0.011343999455372492
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,1023,0.012191999703645706
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,2047,0.013381333400805792
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,2047,0.013232000172138214
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,4095,0.015530666957298914
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,4095,0.029701332251230877
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,8191,0.03136533250411352
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,16383,0.049786667029062905
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,8191,0.017173333714405697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,32767,0.15632533033688864
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,32767,0.08569600184758504
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,8191,0.04825599988301595
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,16383,0.08479467034339905
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,16383,0.031685332457224526
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,1,0.03769599894682566
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,1,0.03173333406448364
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,3,0.03782399992148081
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,3,0.03254933406909307
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,7,0.03775999943415324
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,7,0.03203733265399933
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,15,0.031557333966096245
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,15,0.03811199963092804
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,31,0.037615999579429626
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,31,0.03166933357715607
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,63,0.03782399992148081
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,63,0.03266133368015289
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,127,0.0399893323580424
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,127,0.032186667124430336
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,1,0.048394665122032166
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,3,0.05808533231417338
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,3,0.048842668533325195
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,7,0.04993066688378652
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,7,0.057402665416399636
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,15,0.056874667604764305
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,15,0.04953599969546
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,31,0.0565226674079895
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,31,0.04833066463470459
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,63,0.05816533168156942
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,1,0.056736002365748085
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,63,0.0491946687301
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,127,0.06218666831652323
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,127,0.05294933418432871
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,1,0.09522133072217305
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,1,0.08476799726486206
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,3,0.09519466757774353
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,3,0.08461333314577739
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,7,0.09526933232943217
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,15,0.09523733456929524
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,15,0.08360532919565837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,31,0.09622933467229207
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,7,0.08461333314577739
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,31,0.08362666765848796
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,63,0.0962666670481364
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,63,0.08398399750391643
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,127,0.10338667035102844
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,127,0.08501866459846497
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,float16,1,0.06853866577148438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,1,0.054666668176651
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,float16,3,0.0686773310105006
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,3,0.05537599821885427
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,float16,7,0.06831466654936473
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,7,0.05602133274078369
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,15,0.05539200206597646
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,float16,15,0.06843199829260509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,31,0.05429333448410034
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,float16,63,0.06832533578077953
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,float16,1,0.10129599769910176
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,63,0.05615466833114624
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,float16,31,0.0687360018491745
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,1,0.09328533212343852
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,float16,3,0.10340266426404317
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,3,0.09325333436330159
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,float16,7,0.1027786632378896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,7,0.09460799892743428
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,float16,15,0.10240000486373901
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,15,0.09391466776529948
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,float16,31,0.10379733641942342
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,31,0.09312533338864644
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,float16,63,0.10287466645240784
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,63,0.09402133027712505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,float16,1,0.17750400304794312
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,1,0.15527466932932535
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,3,0.15533333023389181
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,7,0.1567359964052836
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,float16,7,0.17773334185282388
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,1,0.009514666472872099
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,1,0.01007466639081637
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,float16,3,0.1778986652692159
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,3,0.00984533317387104
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,7,0.009381333366036415
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,3,0.011301333705584208
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,7,0.010527999450763067
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,15,0.010559999694426855
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,15,0.009301333377758661
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,31,0.009935999910036722
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,31,0.009328000247478485
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,63,0.009930666536092758
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,63,0.00916800027092298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,127,0.011007999380429586
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,127,0.00922133338948091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,255,0.010965333630641302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,255,0.009338666374484697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,511,0.010981333752473196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,float16,15,0.17951999107996622
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,511,0.011237333218256632
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,1023,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,31,0.15530666708946228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,2047,0.013258667041858038
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,2047,0.013050666699806849
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,1023,0.011215999722480774
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,float16,63,0.17778132359186807
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,float16,31,0.17810134092966715
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,4095,0.015557333827018738
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,15,0.15660799543062845
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,4095,0.015098666151364645
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,8191,0.030805334448814392
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,16383,0.0498986691236496
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,1,0.009663999701539675
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,3,0.00914666677514712
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,8191,0.017605333278576534
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,7,0.010928000013033548
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,7,0.009557333464423815
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,15,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,15,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,31,0.009898666913310686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,31,0.009338666374484697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,63,0.009141333401203156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,63,0.009296000003814697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,1,0.009205333267649015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,127,0.010944000134865442
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,127,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,63,0.15641066431999207
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,255,0.011077333241701126
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,511,0.011039999624093374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,511,0.011237333218256632
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,1023,0.011306667079528173
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,1023,0.01139733319481214
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,2047,0.013066666821638743
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,4095,0.02976000060637792
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,4095,0.015696000307798386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,8191,0.048656001687049866
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,16383,0.08458133538564046
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,16383,0.04993600149949392
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,1,0.011146667102972666
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,2047,0.013429333766301474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,1,0.011018666128317514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,3,0.00955200009047985
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,3,0.009759999811649323
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,7,0.010565333068370819
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,15,0.00933333362142245
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,15,0.010879999647537867
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,31,0.009679999823371569
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,31,0.010346666599313417
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,63,0.011616000284751257
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,3,0.01109333336353302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,63,0.010239999741315842
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,127,0.01121066634853681
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,127,0.01110400011142095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,255,0.009408000235756239
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,255,0.010944000134865442
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,511,0.011183999478816986
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,16383,0.03173866619666418
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,1023,0.013408000270525614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,2047,0.029370665550231934
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,8191,0.03138133386770884
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,2047,0.01524266724785169
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,4095,0.03049066662788391
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,8191,0.08416000008583069
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,8191,0.04867733518282572
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,7,0.009653333574533463
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,16383,0.08530132969220479
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,1,0.009285333255926767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,1,0.01091733326514562
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,3,0.01009599988659223
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,3,0.011136000355084738
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,7,0.00997866690158844
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,255,0.01108266661564509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,7,0.010992000500361124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,15,0.009850666547815004
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,511,0.011125333607196808
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,15,0.01110400011142095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,31,0.009226666763424873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,31,0.00922133338948091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,63,0.011157333850860596
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,127,0.008949333180983862
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,127,0.010944000134865442
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,255,0.010938666760921478
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,255,0.010911999891201654
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,4095,0.04780266682306925
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,511,0.011066666493813196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,511,0.011215999722480774
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,1023,0.011247999966144562
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,2047,0.012975999464591345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,1023,0.01138666644692421
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,2047,0.01292266696691513
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,4095,0.015178666760524115
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,4095,0.015141333142916361
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,8191,0.01926933353145917
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,16383,0.15492266416549683
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,16383,0.04978133241335551
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,1,0.010399999717871347
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,1,0.008997333546479544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,16383,0.031957333286603294
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,3,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,3,0.010933333386977514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,7,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,7,0.009882666791478792
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,15,0.009173333023985228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,15,0.009189333145817121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,31,0.009898666913310686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,31,0.009050666665037474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,63,0.009002666920423508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,63,0.009119999905427298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,1023,0.013290667285521826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,127,0.01102399950226148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,255,0.009008000294367472
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,255,0.009242666885256767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,511,0.011077333241701126
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,511,0.01109333336353302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,63,0.009082666908701261
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,1023,0.011087999989589056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,2047,0.01341333364446958
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,4095,0.030965333183606465
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,2047,0.013301332791646322
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,4095,0.015210667004187902
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,8191,0.049546668926874794
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,16383,0.08673600355784099
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,1,0.011141333729028702
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,1,0.009999999776482582
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,16383,0.050016000866889954
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,3,0.01116266722480456
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,3,0.010832000523805618
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,7,0.010944000134865442
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,7,0.010602666685978571
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,8191,0.03148266673088074
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,15,0.01097600037852923
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,15,0.010879999647537867
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,31,0.010970667004585266
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,31,0.010816000401973724
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,63,0.009930666536092758
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,63,0.01091733326514562
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,127,0.011066666493813196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,127,0.011066666493813196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,255,0.011029332876205444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,255,0.009674666449427605
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,511,0.011183999478816986
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,511,0.011264000087976456
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,1023,0.012986666212479273
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,1023,0.01310933381319046
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,2047,0.029461334149042766
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,127,0.00927466650803884
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,2047,0.015103999525308609
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,4095,0.048197334011395775
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,8191,0.04948799808820089
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,1023,0.011242666592200598
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,16383,0.08630399902661641
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,16383,0.1563146710395813
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,1,0.011168000598748526
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,1,0.010970667004585266
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,3,0.010954666882753372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,3,0.011071999867757162
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,8191,0.031290667752424874
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,7,0.011077333241701126
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,7,0.011178666104873022
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,15,0.011765333513418833
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,15,0.010901333143313726
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,31,0.011338666081428528
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,31,0.011066666493813196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,63,0.011231999844312668
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,63,0.011402666568756104
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,127,0.011130666981140772
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,127,0.011226666470368704
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,255,0.011359999577204386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,255,0.011141333729028702
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,511,0.014890667051076889
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,511,0.01293333371480306
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,1023,0.029477333029111225
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,1023,0.015184000134468079
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,2047,0.04896533489227295
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,2047,0.03133866687615713
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,4095,0.049914668003718056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,4095,0.08473066488901775
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,4095,0.031210665901501972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,8191,0.15745066603024802
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,8191,0.08668800195058186
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,8191,0.08453333377838135
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,1,0.011109333485364914
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,3,0.009045333291093508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,3,0.008922666932145754
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,7,0.009317333499590555
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,16383,0.30404265721638996
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,15,0.008896000062425932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,16383,0.15967466433842978
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,31,0.009194666519761086
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,63,0.00890666681031386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,127,0.00915733352303505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,63,0.009216000015536943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,127,0.009136000027259191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,255,0.008992000172535578
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,255,0.009509333098928133
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,511,0.01090666651725769
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,511,0.01118933285276095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,1023,0.011077333241701126
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,1023,0.009733333562811216
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,2047,0.011002667248249054
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,2047,0.01116266722480456
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,4095,0.01128000020980835
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,4095,0.011242666592200598
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,8191,0.015119999647140503
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,8191,0.015189333508412043
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,16383,0.01924266666173935
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,32767,0.02771199991305669
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,16383,0.01904533306757609
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,32767,0.028069332242012024
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,65535,0.03155199935038885
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,65535,0.03169066707293192
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,1,0.008938666433095932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,1,0.00895999992887179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,3,0.009002666920423508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,3,0.00916800027092298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,7,0.008997333546479544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,7,0.009194666519761086
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,15,0.009189333145817121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,15,0.008965333302815756
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,31,0.008912000184257826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,31,0.009008000294367472
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,63,0.008986666798591614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,63,0.00926399976015091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,127,0.008986666798591614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,255,0.009045333291093508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,1,0.009098666409651438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,255,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,511,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,511,0.01098666712641716
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,1023,0.009253333633144697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,7,0.009418666362762451
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,2047,0.00927466650803884
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,2047,0.010741333166758219
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,4095,0.010890666395425797
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,15,0.00895999992887179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,4095,0.011146667102972666
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,8191,0.015114666273196539
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,8191,0.015125333021084467
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,16383,0.019461333751678467
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,16383,0.019130667050679524
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,32767,0.02342933416366577
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,31,0.009008000294367472
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,32767,0.021338666478792827
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,65535,0.024405332903067272
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,65535,0.023381332556406658
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,1,0.008992000172535578
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,1,0.00901333304742972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,3,0.009093333035707474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,3,0.009152000149091085
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,7,0.00927466650803884
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,15,0.009152000149091085
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,15,0.00903466654320558
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,31,0.009125333279371262
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,31,0.008986666798591614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,63,0.00891733355820179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,127,0.009183999771873156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,127,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,255,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,255,0.010053333515922228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,511,0.009242666885256767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,511,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,1023,0.010522666076819101
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,1023,0.009450666606426239
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,2047,0.009258666386206945
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,2047,0.01116266722480456
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,4095,0.011077333241701126
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,8191,0.015173333386580149
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,8191,0.01532799998919169
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,127,0.00891733355820179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,16383,0.019306667149066925
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,16383,0.017349333812793095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,32767,0.019253333409627277
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,1023,0.009141333401203156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,65535,0.03712533414363861
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,65535,0.021359999974568684
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,1,0.009093333035707474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,3,0.009109333157539368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,3,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,7,0.008992000172535578
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,7,0.009365333244204521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,7,0.009119999905427298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,15,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,63,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,15,0.010773333410422007
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,31,0.00927466650803884
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,31,0.00903466654320558
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,63,0.009008000294367472
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,63,0.00922133338948091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,127,0.00983466642598311
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,127,0.00901333304742972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,255,0.009136000027259191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,255,0.00915733352303505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,511,0.010981333752473196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,511,0.010992000500361124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,1023,0.011168000598748526
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,1023,0.009279999881982803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,4095,0.01098666712641716
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,2047,0.01102399950226148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,4095,0.01314666618903478
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,4095,0.013183999806642532
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,8191,0.015237333873907724
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,8191,0.015103999525308609
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,16383,0.01716800034046173
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,16383,0.01721599946419398
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,32767,0.01939733326435089
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,1,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,32767,0.020479999482631683
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,65535,0.035461333890755974
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,65535,0.05124266445636749
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,1,0.009349333122372627
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,3,0.010384000216921171
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,7,0.009338666374484697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,15,0.009317333499590555
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,7,0.009002666920423508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,1,0.009066666786869368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,15,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,31,0.009253333633144697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,31,0.010832000523805618
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,63,0.009061333412925402
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,63,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,127,0.01102399950226148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,255,0.010255999863147736
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,255,0.00890666681031386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,511,0.010869332899649939
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,511,0.010890666395425797
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,1023,0.010992000500361124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,1023,0.009621333330869675
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,2047,0.011157333850860596
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,2047,0.010933333386977514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,4095,0.011231999844312668
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,4095,0.01101333275437355
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,8191,0.015322666615247726
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,8191,0.01515199989080429
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,16383,0.021231998999913532
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,16383,0.021162666380405426
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,32767,0.02345066765944163
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,32767,0.023050665855407715
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,65535,0.025685332715511322
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,65535,0.025386666258176167
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,1,0.009242666885256767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,1,0.008976000050703684
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,3,0.009269333134094873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,3,0.008943999807039896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,7,0.009136000027259191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,7,0.009578666960199675
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,15,0.009349333122372627
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,15,0.008992000172535578
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,31,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,31,0.009914666414260864
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,63,0.008922666932145754
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,127,0.009061333412925402
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,127,0.009050666665037474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,255,0.00892800030608972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,255,0.009109333157539368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,511,0.011098666737476984
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,511,0.010944000134865442
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,32767,0.03352533280849457
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,1023,0.009152000149091085
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,1023,0.00955200009047985
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,3,0.00897066667675972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,2047,0.011157333850860596
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,2047,0.010847999403874079
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,4095,0.01102399950226148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,4095,0.011018666128317514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,8191,0.015311999867359797
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,8191,0.015279999623696009
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,16383,0.01794133335351944
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,16383,0.017157333592573803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,32767,0.020538666596015293
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,127,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,32767,0.019071999937295914
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,65535,0.03543466577927271
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,65535,0.02109333376089732
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,1,0.009056000038981438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,1,0.008879999940594038
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,3,0.00914666677514712
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,3,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,7,0.008896000062425932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,7,0.009061333412925402
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,15,0.010175999874869982
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,15,0.009098666409651438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,31,0.008853333070874214
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,31,0.008938666433095932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,63,0.009008000294367472
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,63,0.008943999807039896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,127,0.009189333145817121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,127,0.00898133342464765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,255,0.009279999881982803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,255,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,511,0.009194666519761086
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,511,0.009898666913310686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,1023,0.009216000015536943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,1023,0.009599999835093817
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,2047,0.010869332899649939
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,2047,0.011029332876205444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,4095,0.01321600005030632
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,8191,0.01515199989080429
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,8191,0.015040000279744467
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,2047,0.010965333630641302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,16383,0.017312000195185345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,63,0.009205333267649015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,16383,0.016234666109085083
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,32767,0.0335359995563825
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,32767,0.01912533367673556
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,65535,0.052005335688591
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,65535,0.03448000053564707
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,1,0.00927466650803884
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,1,0.009018666421373686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,3,0.009258666386206945
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,3,0.009093333035707474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,7,0.009509333098928133
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,15,0.00898133342464765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,15,0.009093333035707474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,31,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,63,0.009285333255926767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,63,0.00898133342464765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,127,0.009162666896979014
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,127,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,255,0.008965333302815756
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,255,0.00895999992887179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,511,0.011002667248249054
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,511,0.010608000059922537
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,1023,0.010863999525705973
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,1023,0.00903466654320558
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,2047,0.012986666212479273
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,2047,0.012144000579913458
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,4095,0.015024000157912573
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,4095,0.013290667285521826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,8191,0.01716800034046173
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,8191,0.015109332899252573
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,16383,0.017840000490347546
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,16383,0.03183999905983607
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,32767,0.04983466863632202
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,32767,0.03323200096686681
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,65535,0.052144000927607216
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,65535,0.08667199810345967
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,4095,0.013295999417702356
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,1,0.009178666397929192
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,1,0.009269333134094873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,3,0.009253333633144697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,3,0.010992000500361124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,7,0.01097600037852923
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,7,0.011061333119869232
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,15,0.00926399976015091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,15,0.00916800027092298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,31,0.009306666751702627
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,63,0.010885333021481832
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,31,0.00914666677514712
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,63,0.010741333166758219
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,127,0.010389333590865135
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,31,0.009098666409651438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,127,0.009183999771873156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,255,0.009317333499590555
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,255,0.010837333897749582
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,511,0.011306667079528173
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,1023,0.013066666821638743
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,1023,0.013162666310866674
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,2047,0.015002666662136713
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,4095,0.03142933299144109
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,4095,0.01752000053723653
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,8191,0.04993066688378652
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,8191,0.03218133250872294
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,3,0.01081066702802976
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,3,0.009557333464423815
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,1,0.011018666128317514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,1,0.010629333555698395
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,7,0.011039999624093374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,7,0.009583999713261923
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,15,0.00996800015370051
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,15,0.010960000256697336
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,31,0.01110400011142095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,31,0.010965333630641302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,63,0.01098666712641716
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,63,0.011002667248249054
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,7,0.009056000038981438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,127,0.01098666712641716
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,127,0.011045332998037338
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,255,0.010255999863147736
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,255,0.011136000355084738
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,511,0.011258666714032492
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,511,0.011029332876205444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,2047,0.015184000134468079
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,1023,0.013327999661366144
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,1023,0.013066666821638743
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,2047,0.02935466667016347
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,2047,0.015295999745527903
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,4095,0.048581331968307495
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,4095,0.03148799886306127
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,8191,0.08598933617273967
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,8191,0.04980266590913137
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,1,0.01138666644692421
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,3,0.011461333682139715
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,3,0.011253333340088526
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,7,0.01128000020980835
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,1,0.011018666128317514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,7,0.010960000256697336
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,15,0.011424000064531961
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,31,0.010922666639089584
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,31,0.011231999844312668
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,63,0.011125333607196808
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,63,0.01121066634853681
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,127,0.010992000500361124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,127,0.01126933346192042
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,255,0.011039999624093374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,255,0.01102399950226148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,511,0.01509333277742068
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,511,0.013088000317414602
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,1023,0.030026666820049286
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,511,0.011141333729028702
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,1023,0.015765332927306492
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,2047,0.04808000226815542
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,2047,0.03169066707293192
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,4095,0.04937600096066793
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,4095,0.08523733417193095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,1,0.013295999417702356
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,8191,0.08745066324869792
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,3,0.015109332899252573
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,1,0.014378666877746582
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,3,0.013397333522637686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,8191,0.15777599811553955
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,7,0.01498666654030482
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,7,0.013397333522637686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,15,0.015018666783968607
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,15,0.01322666679819425
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,31,0.015072000523408255
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,15,0.011125333607196808
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,31,0.013183999806642532
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,63,0.013338666409254074
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,127,0.014922666052977243
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,63,0.015040000279744467
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,127,0.013077333569526672
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,255,0.01504533365368843
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,1023,0.04915200173854828
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,255,0.013013333082199097
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,1023,0.031343999008337654
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,2047,0.08504000306129456
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,2047,0.05093333125114441
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,511,0.01724799970785777
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,511,0.03046400099992752
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,1,0.00915733352303505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,1,0.008890666688481966
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,7,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,4095,0.08690133690834045
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,7,0.00898133342464765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,4095,0.15949333707491556
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,8191,0.16036799550056458
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,15,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,31,0.00895999992887179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,31,0.009018666421373686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,15,0.009082666908701261
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,3,0.010847999403874079
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,63,0.00933333362142245
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,63,0.009183999771873156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,127,0.009365333244204521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,8191,0.3049173355102539
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,255,0.011039999624093374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,511,0.010922666639089584
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,511,0.011002667248249054
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,1023,0.011125333607196808
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,1023,0.010794666906197866
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,2047,0.010965333630641302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,2047,0.011152000476916632
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,4095,0.01101333275437355
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,4095,0.01116266722480456
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,8191,0.017093333105246227
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,8191,0.01532799998919169
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,16383,0.01922133316596349
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,16383,0.019285333653291065
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,32767,0.021551998953024547
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,32767,0.020901332298914593
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,65535,0.036570665736993156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,65535,0.023397333920001984
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,1,0.009279999881982803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,1,0.00895999992887179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,3,0.008986666798591614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,3,0.008986666798591614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,7,0.009455999980370203
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,7,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,15,0.008938666433095932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,15,0.00914666677514712
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,31,0.00897066667675972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,31,0.009077333534757296
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,63,0.008912000184257826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,63,0.009109333157539368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,127,0.01009599988659223
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,255,0.009066666786869368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,255,0.008853333070874214
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,511,0.010954666882753372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,511,0.011109333485364914
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,1023,0.009050666665037474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,1023,0.009152000149091085
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,2047,0.011205332974592844
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,4095,0.01320533330241839
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,4095,0.013013333082199097
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,8191,0.015066667149464289
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,3,0.009685333197315535
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,127,0.00897066667675972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,16383,0.017242666333913803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,16383,0.017077332983414333
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,255,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,32767,0.019226666539907455
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,65535,0.050714666644732155
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,65535,0.03549866626660029
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,1,0.009093333035707474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,1,0.008933333059151968
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,3,0.009290666629870733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,3,0.00922133338948091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,7,0.009205333267649015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,15,0.009082666908701261
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,15,0.009322666873534521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,31,0.009130666653315226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,31,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,63,0.009242666885256767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,127,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,255,0.009392000113924345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,255,0.009573333586255709
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,511,0.011050666371981302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,2047,0.010944000134865442
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,511,0.011071999867757162
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,8191,0.01525866612792015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,1023,0.00933333362142245
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,1023,0.011050666371981302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,2047,0.013157332936922709
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,2047,0.011354666203260422
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,32767,0.03364266703526179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,4095,0.015184000134468079
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,4095,0.013845333208640417
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,8191,0.01720000058412552
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,8191,0.015103999525308609
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,16383,0.031680000325044
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,16383,0.017914666483799618
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,7,0.00916800027092298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,32767,0.033957332372665405
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,65535,0.08636266986529033
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,65535,0.053360000252723694
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,1,0.008922666932145754
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,3,0.009397333487868309
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,63,0.009183999771873156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,3,0.00892800030608972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,7,0.008858666444818178
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,7,0.009082666908701261
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,15,0.009029333169261614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,15,0.00927466650803884
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,127,0.009162666896979014
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,31,0.009072000160813332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,31,0.00891733355820179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,63,0.00916800027092298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,63,0.008997333546479544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,127,0.009216000015536943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,127,0.009258666386206945
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,255,0.009119999905427298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,255,0.009509333098928133
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,511,0.01089599976936976
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,511,0.011194666226704916
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,1023,0.010949333508809408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,1023,0.011029332876205444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,2047,0.013221333424250284
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,2047,0.011237333218256632
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,4095,0.015125333021084467
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,4095,0.013274667163689932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,8191,0.029866665601730347
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,8191,0.017008000363906223
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,16383,0.03148266673088074
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,16383,0.04791999856630961
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,32767,0.049072002371152244
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,32767,0.08459200461705525
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,32767,0.05120000243186951
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,65535,0.08574400345484416
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,65535,0.15660267074902853
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,1,0.009343999748428663
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,1,0.010911999891201654
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,1,0.011221333096424738
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,3,0.011183999478816986
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,127,0.00927466650803884
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,3,0.011130666981140772
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,7,0.010954666882753372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,7,0.011061333119869232
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,15,0.01099733387430509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,15,0.011157333850860596
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,31,0.011039999624093374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,31,0.01090666651725769
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,63,0.010762666662534079
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,63,0.01232533281048139
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,127,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,127,0.010933333386977514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,255,0.01128000020980835
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,255,0.011077333241701126
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,511,0.011359999577204386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,511,0.011120000233252844
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,1023,0.015087999403476715
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,1023,0.013343999783198038
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,2047,0.03030933439731598
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,2047,0.01729600007335345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,1,0.010922666639089584
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,4095,0.031370667119820915
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,4095,0.04846400022506714
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,3,0.01137599969903628
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,7,0.011328000575304031
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,15,0.012170666207869848
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,7,0.011274666835864386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,31,0.011152000476916632
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,63,0.01119999960064888
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,63,0.011178666104873022
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,15,0.011061333119869232
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,31,0.011285333583752314
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,127,0.01116266722480456
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,127,0.01116266722480456
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,255,0.011296000331640244
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,255,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,511,0.014730667074521383
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,511,0.013072000195582708
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,1023,0.03019733230272929
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,1023,0.0162773331006368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,1,0.011215999722480774
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,3,0.010992000500361124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,2047,0.03202133377393087
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,4095,0.08685333530108134
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,3,0.014917333920796713
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,1,0.014074667046467463
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,1,0.013183999806642532
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,3,0.012901333471139273
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,7,0.013839999834696451
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,15,0.014885333677132925
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,7,0.015008000036080679
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,15,0.013183999806642532
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,31,0.015141333142916361
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,31,0.013002666334311167
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,63,0.014970666418472925
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,63,0.013269333789745966
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,127,0.014912000546852747
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,127,0.013317332913478216
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,255,0.014522666732470194
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,255,0.013258667041858038
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,511,0.031173333525657654
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,2047,0.04980800052483877
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,511,0.017125333348910015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,4095,0.05064533154169718
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,2047,0.08517866333325703
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,2047,0.050794666012128196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,1023,0.031717332700888314
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,1,0.01727466657757759
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,4095,0.15901333093643188
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,3,0.01933866615096728
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,3,0.01727466657757759
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,1,0.019317333896954853
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,7,0.017535999417304993
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,4095,0.08736532926559448
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,15,0.01911466692884763
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,31,0.019386666516462963
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,7,0.019381333142518997
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,31,0.017360000560681026
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,63,0.019482667247454327
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,127,0.019248000035683315
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,63,0.017338667064905167
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,127,0.017258666455745697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,255,0.017349333812793095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,1023,0.0491839994986852
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,255,0.029792000850041706
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,1023,0.08430400490760803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,511,0.04894400139649709
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,511,0.03147733211517334
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,1023,0.0506879985332489
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,2047,0.08795199791590373
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,15,0.017162666966517765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,2047,0.15772799650828043
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,4095,0.29971200227737427
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,1,0.01312000056107839
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,4095,0.16245333353678384
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,1,0.012794667234023413
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,3,0.013056000073750814
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,7,0.013130666067202887
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,3,0.013183999806642532
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,7,0.0129120002190272
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,15,0.013066666821638743
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,15,0.013173333058754602
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,31,0.013082666943470636
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,31,0.01309866706530253
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,63,0.012901333471139273
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,127,0.013141332815090815
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,127,0.013023999830087027
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,255,0.013157332936922709
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,511,0.015066667149464289
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,255,0.01301866645614306
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,1023,0.031343999008337654
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,1,0.015295999745527903
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,1023,0.01725333308180173
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,1,0.01340266689658165
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,3,0.015157333264748255
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,3,0.013199999928474426
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,7,0.014101333916187286
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,7,0.01515199989080429
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,15,0.014896000425020853
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,63,0.013290667285521826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,15,0.013141332815090815
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,31,0.015135999768972397
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,31,0.013295999417702356
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,63,0.015103999525308609
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,63,0.013349333157142004
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,511,0.015205333630243937
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,127,0.015082667271296183
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,127,0.013269333789745966
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,255,0.015141333142916361
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,255,0.01310933381319046
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,511,0.030906667311986286
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,511,0.01727466657757759
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,1,0.0174346665541331
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,1023,0.03181333343187968
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,1023,0.050026665131251015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,3,0.019440000255902607
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,3,0.017456000049908955
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,7,0.019237333287795384
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,7,0.01717866708834966
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,15,0.01939733326435089
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,31,0.017312000195185345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,63,0.01924266666173935
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,15,0.017312000195185345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,31,0.019146667172511418
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,63,0.01735466718673706
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,127,0.019082666685183842
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,127,0.017312000195185345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,1,0.019199999670187633
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,255,0.017237332959969837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,255,0.03142400085926056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,511,0.032357332607110344
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,511,0.04875733455022176
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,1,0.025434667865435284
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,1,0.029637334247430164
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,3,0.029477333029111225
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,3,0.02606400102376938
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,1023,0.05083733300367991
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,1023,0.08483200271924336
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,15,0.029535998900731403
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,15,0.025568000972270966
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,7,0.029306667546431225
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,31,0.029557332396507263
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,127,0.03348266581694285
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,127,0.02613866577545802
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,255,0.0497920016447703
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,255,0.03372266640265783
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,63,0.025610665480295818
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,7,0.025674665967623394
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,63,0.0296426663796107
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,31,0.02569599946339925
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,1023,0.1574613352616628
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,511,0.05136000116666158
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,511,0.0853706697622935
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,1023,0.08717866738637288
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,1,0.014965333044528961
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,1,0.015205333630243937
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,3,0.01682666689157486
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,3,0.015082667271296183
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,7,0.01691199963291486
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,15,0.015130666395028433
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,31,0.016906666258970898
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,31,0.015205333630243937
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,63,0.015216000378131866
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,63,0.015135999768972397
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,7,0.015247999380032221
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,127,0.016469333320856094
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,127,0.015029333531856537
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,255,0.01716800034046173
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,255,0.01533866673707962
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,511,0.03140799949566523
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,511,0.019173332800467808
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,15,0.015024000157912573
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,1,0.01821333294113477
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,3,0.019978666057189304
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,3,0.017477333545684814
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,7,0.01921066641807556
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,7,0.017792000124851864
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,15,0.01722666621208191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,15,0.019306667149066925
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,1,0.01913600042462349
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,31,0.01929066702723503
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,31,0.017338667064905167
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,63,0.019194666296243668
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,63,0.017231999586025875
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,127,0.019434666881958645
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,127,0.017210666090250015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,255,0.031231999397277832
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,255,0.01730666682124138
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,511,0.049829334020614624
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,1,0.031445334355036415
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,511,0.03356266766786575
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,1,0.027456000447273254
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,3,0.027658666173617046
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,3,0.03143466760714849
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,7,0.027610667049884796
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,7,0.031632001201311745
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,15,0.027701333165168762
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,31,0.027402666707833607
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,15,0.031343999008337654
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,31,0.0316746657093366
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,63,0.031557333966096245
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,63,0.027664000789324444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,127,0.03562666724125544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,127,0.028186666468779247
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,255,0.05189866820971171
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,255,0.03552533437808355
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,1,0.0535093347231547
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,1,0.04427733520666758
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,511,0.0888320008913676
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,3,0.05411200225353241
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,511,0.05413866539796194
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,3,0.044719999035199486
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,7,0.05259733398755392
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,15,0.053082664807637535
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,7,0.04424533247947693
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,15,0.044069334864616394
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,31,0.04491733511288961
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,63,0.04588800172011057
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,127,0.047882666190465294
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,127,0.05629866818586985
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,63,0.05420266588528951
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,1,0.009370666618148485
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,255,0.056458666920661926
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,3,0.009061333412925402
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,255,0.08880000313123067
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,1,0.009216000015536943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,3,0.008943999807039896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,7,0.009050666665037474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,15,0.009306666751702627
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,31,0.008976000050703684
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,31,0.00914666677514712
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,63,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,63,0.009296000003814697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,127,0.00922133338948091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,127,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,31,0.05337599913279215
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,255,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,255,0.010757333288590113
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,511,0.011007999380429586
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,511,0.011077333241701126
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,1023,0.01119999960064888
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,2047,0.01101333275437355
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,2047,0.011066666493813196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,1023,0.01098666712641716
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,4095,0.014906667172908783
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,4095,0.014837333311637243
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,8191,0.01617066686352094
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,8191,0.016314666718244553
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,16383,0.019248000035683315
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,32767,0.03440533330043157
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,16383,0.01911466692884763
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,65535,0.05415999889373779
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,65535,0.03585066646337509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,1,0.008954666554927826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,7,0.009242666885256767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,15,0.009279999881982803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,1,0.009392000113924345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,3,0.009290666629870733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,3,0.009519999846816063
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,7,0.00922133338948091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,7,0.009136000027259191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,15,0.00914666677514712
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,15,0.009888000165422758
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,31,0.009226666763424873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,511,0.09153599540392558
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,511,0.16061333815256754
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,63,0.008901333436369896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,63,0.009648000200589498
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,127,0.009178666397929192
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,255,0.00898133342464765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,511,0.011050666371981302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,511,0.011168000598748526
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,1023,0.009136000027259191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,1023,0.010112000008424124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,2047,0.012917333592971167
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,2047,0.011253333340088526
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,4095,0.014645333091417948
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,4095,0.013936000565687815
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,8191,0.01711999997496605
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,8191,0.015087999403476715
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,16383,0.0317546675602595
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,16383,0.01894933357834816
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,32767,0.05117866893609365
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,32767,0.033370666205883026
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,65535,0.08755200107892354
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,65535,0.0521919975678126
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,1,0.008885333314538002
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,1,0.009039999917149544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,3,0.009018666421373686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,3,0.009205333267649015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,7,0.009375999992092451
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,7,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,15,0.008938666433095932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,31,0.009045333291093508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,31,0.009290666629870733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,31,0.00922133338948091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,63,0.010149333626031876
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,63,0.008922666932145754
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,127,0.009952000031868616
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,127,0.009525333220760027
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,255,0.009072000160813332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,255,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,511,0.011029332876205444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,255,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,127,0.009482666850090027
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,511,0.010821333775917688
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,1023,0.011781333635250727
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,1023,0.010992000500361124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,2047,0.012053333222866058
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,4095,0.01524266724785169
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,4095,0.013183999806642532
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,8191,0.029904000461101532
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,16383,0.04818666477998098
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,16383,0.03137599925200144
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,32767,0.050016000866889954
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,32767,0.08498133222262065
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,65535,0.15717333555221558
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,65535,0.08683199683825175
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,1,0.008976000050703684
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,32767,0.02146133283774058
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,3,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,3,0.00926399976015091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,7,0.00903466654320558
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,15,0.009178666397929192
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,15,0.009136000027259191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,31,0.00922133338948091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,31,0.009029333169261614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,63,0.00901333304742972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,63,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,127,0.009375999992092451
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,127,0.009061333412925402
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,255,0.00933333362142245
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,255,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,2047,0.013130666067202887
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,511,0.011120000233252844
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,511,0.010960000256697336
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,8191,0.017024000485738117
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,1023,0.012741333494583765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,2047,0.013045333325862885
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,2047,0.013264000415802002
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,4095,0.01623999948302905
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,4095,0.030581332743167877
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,8191,0.03145600110292435
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,8191,0.04808000226815542
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,1,0.00984533317387104
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,16383,0.05012799799442291
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,16383,0.08435199658075969
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,7,0.008949333180983862
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,32767,0.08482666810353597
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,32767,0.1562879979610443
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,15,0.009119999905427298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,65535,0.15707733233769736
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,65535,0.30053333441416424
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,1023,0.011077333241701126
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,1,0.023306667804718018
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,1,0.020842666427294414
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,3,0.02329600105683009
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,3,0.019354666272799175
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,7,0.020207999895016353
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,7,0.02350933353106181
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,15,0.023631999890009563
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,15,0.0195573332409064
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,31,0.02327999969323476
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,31,0.020645332833131153
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,63,0.023200000325838726
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,63,0.01922133316596349
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,127,0.023247999449570973
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,127,0.021295999487241108
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,255,0.031925333042939506
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,1,0.03136533250411352
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,255,0.019733333339293797
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,1,0.027744000156720478
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,3,0.02762666592995326
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,3,0.03163733333349228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,7,0.031514666974544525
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,7,0.027488000690937042
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,15,0.03236266722281774
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,15,0.02757333219051361
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,31,0.02762666592995326
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,31,0.03140799949566523
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,63,0.031471999982992806
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,63,0.02773866554101308
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,127,0.027615999182065327
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,127,0.03626133253177007
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,255,0.052149335543314614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,1,0.05766933163007101
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,255,0.03696533292531967
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,3,0.056757330894470215
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,3,0.04863466819127401
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,7,0.047930667797724404
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,7,0.056074668963750206
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,1,0.0481333335240682
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,15,0.05621333420276642
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,15,0.04781866570313772
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,31,0.057392001152038574
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,31,0.04996266464392344
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,63,0.05823466678460439
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,63,0.0481333335240682
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,127,0.06126933296521505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,127,0.052229334910710655
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,255,0.09354666868845622
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,255,0.06137066582838694
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,1,0.09550399581591289
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,1,0.08492799599965413
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,3,0.0956053336461385
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,3,0.08368000388145447
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,7,0.09515200058619182
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,7,0.08328000207742055
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,31,0.09574400385220845
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,31,0.08433600266774495
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,15,0.09518399834632874
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,15,0.08365866541862488
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,63,0.0843946635723114
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,63,0.09676266709963481
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,127,0.08475733796755473
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,127,0.1011306643486023
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,255,0.16660799582799277
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,1,0.03858133405447006
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,1,0.03346133232116699
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,255,0.10207466284434001
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,3,0.037621334195137024
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,3,0.03229333211978277
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,7,0.037946666280428566
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,7,0.031541332602500916
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,15,0.03772799919048945
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,31,0.03782933453718821
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,15,0.031983998914559685
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,31,0.03188266605138779
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,63,0.037578667203585304
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,63,0.03165333221356074
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,127,0.032618666688601174
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,1,0.05527466535568237
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,1,0.04782933493455251
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,3,0.05428266525268555
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,127,0.03958400090535482
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,3,0.04806933303674062
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,7,0.05569600065549215
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,7,0.048021331429481506
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,15,0.056186666091283165
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,15,0.046682665745417275
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,31,0.05562133093674978
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,63,0.055685331424077354
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,31,0.047637333472569786
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,63,0.047872001926104225
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,127,0.06044266621271769
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,127,0.052154665191968284
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,1,0.10157333811124165
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,1,0.09277866284052531
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,3,0.10366933544476827
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,3,0.09290666381518047
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,7,0.101583997408549
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,7,0.0928106705347697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,15,0.10276800394058228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,15,0.09379200140635173
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,31,0.09288533528645833
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,63,0.10239467024803162
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,63,0.09306666254997253
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,1,0.17721599340438843
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,127,0.09489066402117412
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,31,0.10284266869227092
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,127,0.11158399780591328
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,1,0.15622400244077048
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,3,0.1775146722793579
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,3,0.15625600020090738
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,7,0.17729065815607706
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,7,0.15424000223477682
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,15,0.15574933091799417
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,15,0.1772480010986328
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,1,0.009173333023985228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,3,0.010101333260536194
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,31,0.1781066656112671
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,3,0.010709332923094431
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,1,0.009183999771873156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,7,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,7,0.00926399976015091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,15,0.010768000036478043
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,31,0.00895999992887179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,31,0.01109333336353302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,63,0.009114666531483332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,63,0.009050666665037474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,15,0.009056000038981438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,127,0.00926399976015091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,127,0.010010666524370512
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,255,0.009072000160813332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,255,0.010954666882753372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,511,0.011157333850860596
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,511,0.011120000233252844
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,1023,0.010847999403874079
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,1023,0.011055999745925268
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,2047,0.013189333180586496
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,2047,0.013317332913478216
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,31,0.1548426647981008
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,4095,0.015263999501864115
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,4095,0.015253332753976187
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,8191,0.017530667285124462
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,8191,0.017360000560681026
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,32767,0.051856001218159996
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,16383,0.02141333371400833
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,32767,0.03380800038576126
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,1,0.009269333134094873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,1,0.009290666629870733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,16383,0.03340800106525421
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,3,0.009173333023985228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,3,0.00916800027092298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,7,0.00922133338948091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,7,0.00891733355820179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,15,0.009109333157539368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,15,0.00926399976015091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,31,0.009125333279371262
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,63,0.15505066514015198
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,63,0.009050666665037474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,63,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,127,0.009183999771873156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,127,0.009093333035707474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,255,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,255,0.009343999748428663
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,511,0.011018666128317514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,1023,0.011130666981140772
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,1023,0.010992000500361124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,63,0.1790613333384196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,2047,0.013023999830087027
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,4095,0.015423999478419622
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,511,0.009290666629870733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,8191,0.017055999487638474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,16383,0.04842133323351542
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,16383,0.03203733265399933
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,32767,0.08673600355784099
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,4095,0.013546666751305262
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,32767,0.05131733417510986
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,1,0.009152000149091085
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,1,0.011071999867757162
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,3,0.009194666519761086
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,3,0.009194666519761086
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,7,0.010069333637754122
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,7,0.009216000015536943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,31,0.00901333304742972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,15,0.009162666896979014
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,15,0.009194666519761086
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,31,0.009088000282645226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,63,0.00884799969693025
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,63,0.009290666629870733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,127,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,127,0.00916800027092298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,255,0.009018666421373686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,255,0.009322666873534521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,511,0.010944000134865442
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,511,0.010970667004585266
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,2047,0.011109333485364914
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,1023,0.011930666863918304
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,1023,0.011770666887362799
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,127,0.15707199772198996
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,2047,0.013210666676362356
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,2047,0.013280000537633896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,8191,0.029818666477998097
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,127,0.19057599703470865
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,4095,0.0296426663796107
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,8191,0.04825599988301595
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,8191,0.03173333406448364
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,16383,0.08444266517957051
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,1,0.010453333457310995
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,1,0.011029332876205444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,32767,0.1567200024922689
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,3,0.009754666437705358
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,3,0.0107893335322539
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,7,0.010960000256697336
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,15,0.011039999624093374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,15,0.009765333185593287
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,31,0.011109333485364914
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,63,0.011077333241701126
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,63,0.01080000028014183
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,16383,0.049925332268079124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,4095,0.01525866612792015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,32767,0.08737599849700928
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,127,0.011007999380429586
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,127,0.011071999867757162
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,255,0.009216000015536943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,255,0.010879999647537867
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,511,0.011077333241701126
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,511,0.011296000331640244
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,1023,0.013130666067202887
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,2047,0.015365333606799444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,7,0.0107893335322539
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,2047,0.029440000653266907
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,4095,0.047872001926104225
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,31,0.009194666519761086
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,4095,0.02976000060637792
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,8191,0.04863999783992767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,8191,0.08318399886290233
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,31,0.01051733394463857
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,16383,0.15641599893569946
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,1,0.009514666472872099
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,1,0.010885333021481832
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,16383,0.08474666873613994
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,3,0.008954666554927826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,7,0.009445333232482275
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,3,0.009450666606426239
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,15,0.00897066667675972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,15,0.01003200002014637
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,31,0.00938666673998038
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,31,0.009269333134094873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,63,0.008976000050703684
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,63,0.009488000224033991
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,127,0.011061333119869232
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,127,0.011039999624093374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,255,0.00901333304742972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,255,0.009946666657924652
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,511,0.011173332730929056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,511,0.011231999844312668
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,1023,0.011237333218256632
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,1023,0.010965333630641302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,2047,0.013151999562978745
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,2047,0.013045333325862885
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,1023,0.013290667285521826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,4095,0.015130666395028433
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,4095,0.015061333775520325
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,8191,0.03127466638882955
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,8191,0.01918399954835574
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,16383,0.050016000866889954
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,16383,0.03330666571855545
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,1,0.009653333574533463
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,1,0.010778666784365972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,3,0.008992000172535578
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,3,0.009839999799927076
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,7,0.009269333134094873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,7,0.010938666760921478
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,15,0.009077333534757296
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,15,0.009008000294367472
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,31,0.009258666386206945
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,31,0.009546666716535887
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,63,0.009946666657924652
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,63,0.009103999783595404
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,127,0.009098666409651438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,32767,0.15776532888412476
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,127,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,32767,0.2993280092875163
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,255,0.009258666386206945
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,255,0.013461332768201828
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,511,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,511,0.01116266722480456
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,1023,0.013104000439246496
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,2047,0.01328533391157786
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,4095,0.030752000709374745
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,8191,0.048122664292653404
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,8191,0.031557333966096245
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,16383,0.08498666683832805
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,16383,0.05011733373006185
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,1,0.009712000067035357
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,1,0.0102613332370917
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,3,0.009599999835093817
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,3,0.01099733387430509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,7,0.01007466639081637
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,7,0.009695999945203463
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,15,0.009759999811649323
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,15,0.010922666639089584
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,31,0.009925333162148794
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,31,0.010069333637754122
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,63,0.010533332824707031
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,63,0.011007999380429586
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,127,0.011018666128317514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,127,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,255,0.010703999549150467
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,255,0.010970667004585266
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,511,0.01109333336353302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,511,0.011370666325092316
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,7,0.011098666737476984
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,1023,0.012965332716703415
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,1023,0.014384000251690546
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,2047,0.029509333272775013
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,2047,0.015333333363135656
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,4095,0.04785599807898203
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,2047,0.013130666067202887
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,4095,0.015200000256299973
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,4095,0.030389333764712017
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,8191,0.0846613347530365
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,1,0.009119999905427298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,1,0.00897066667675972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,3,0.009008000294367472
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,3,0.009152000149091085
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,7,0.009733333562811216
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,16383,0.1567200024922689
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,7,0.009312000125646591
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,15,0.009999999776482582
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,16383,0.08562133709589641
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,31,0.009050666665037474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,63,0.009077333534757296
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,63,0.008976000050703684
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,127,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,127,0.008949333180983862
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,255,0.009183999771873156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,255,0.00892800030608972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,511,0.010954666882753372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,511,0.010949333508809408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,1023,0.009706666693091393
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,1023,0.011125333607196808
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,2047,0.01102399950226148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,2047,0.010960000256697336
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,4095,0.011061333119869232
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,4095,0.011045332998037338
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,8191,0.014917333920796713
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,8191,0.01525866612792015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,16383,0.019050666441520054
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,16383,0.021210665504137676
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,32767,0.02752533306678136
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,32767,0.027274665733178455
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,65535,0.03155199935038885
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,65535,0.0315733328461647
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,1,0.00895999992887179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,1,0.00898133342464765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,3,0.009088000282645226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,3,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,7,0.00895999992887179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,1023,0.011173332730929056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,7,0.009205333267649015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,15,0.009082666908701261
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,15,0.009002666920423508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,31,0.00927466650803884
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,31,0.009103999783595404
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,63,0.00890666681031386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,63,0.008992000172535578
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,8191,0.04934399823347727
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,127,0.009045333291093508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,127,0.009125333279371262
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,255,0.008896000062425932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,255,0.008901333436369896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,511,0.010970667004585266
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,511,0.011077333241701126
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,1023,0.009359999870260557
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,2047,0.010138666878143946
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,15,0.01091733326514562
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,8191,0.015237333873907724
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,31,0.009189333145817121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,8191,0.01524266724785169
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,16383,0.019274666905403137
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,32767,0.023333333432674408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,32767,0.021733333667119343
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,65535,0.025333332518736523
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,65535,0.02345066765944163
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,1,0.009408000235756239
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,1,0.009205333267649015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,3,0.008896000062425932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,3,0.008976000050703684
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,7,0.009039999917149544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,7,0.009216000015536943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,15,0.00892800030608972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,15,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,31,0.00914666677514712
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,31,0.00897066667675972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,63,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,63,0.008965333302815756
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,127,0.008938666433095932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,127,0.009712000067035357
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,255,0.009279999881982803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,255,0.009183999771873156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,511,0.011087999989589056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,511,0.010938666760921478
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,1023,0.009381333366036415
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,1023,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,2047,0.010981333752473196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,2047,0.011029332876205444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,4095,0.011007999380429586
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,4095,0.011066666493813196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,1023,0.009130666653315226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,8191,0.015386667102575302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,2047,0.010954666882753372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,4095,0.010911999891201654
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,16383,0.01738133281469345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,16383,0.01937599976857503
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,32767,0.018986667195955913
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,1,0.009152000149091085
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,65535,0.02109333376089732
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,1,0.009189333145817121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,3,0.009296000003814697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,3,0.009002666920423508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,7,0.008992000172535578
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,7,0.010954666882753372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,15,0.009066666786869368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,15,0.010965333630641302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,31,0.009056000038981438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,31,0.01089599976936976
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,63,0.009119999905427298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,63,0.01090666651725769
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,127,0.00898133342464765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,127,0.009898666913310686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,255,0.009050666665037474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,255,0.009877333417534828
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,511,0.01090666651725769
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,511,0.010944000134865442
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,1023,0.010821333775917688
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,1023,0.010933333386977514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,2047,0.011050666371981302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,8191,0.015040000279744467
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,2047,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,4095,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,4095,0.01109333336353302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,4095,0.011194666226704916
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,8191,0.016415999581416447
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,8191,0.016165333489576977
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,16383,0.01918399954835574
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,16383,0.019472000499566395
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,32767,0.02083733429511388
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,32767,0.02348800003528595
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,32767,0.02325333406527837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,65535,0.025407999753952026
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,65535,0.025279998779296875
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,1,0.00891733355820179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,1,0.009973333527644476
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,3,0.00914666677514712
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,3,0.00926399976015091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,7,0.00890666681031386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,7,0.009002666920423508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,15,0.00922133338948091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,15,0.008976000050703684
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,31,0.009119999905427298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,31,0.00915733352303505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,63,0.008858666444818178
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,127,0.00901333304742972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,127,0.009056000038981438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,255,0.00914666677514712
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,255,0.009136000027259191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,511,0.00898133342464765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,511,0.0107893335322539
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,1023,0.009354666496316591
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,1023,0.010842667271693548
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,2047,0.009989333028594652
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,2047,0.009578666960199675
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,4095,0.01090666651725769
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,4095,0.011130666981140772
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,8191,0.015237333873907724
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,8191,0.015098666151364645
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,16383,0.01756799966096878
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,16383,0.017301333447297413
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,32767,0.02107200026512146
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,32767,0.01916266605257988
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,65535,0.021242665747801464
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,65535,0.0369759996732076
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,1,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,1,0.008912000184257826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,3,0.00926399976015091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,7,0.010405333091815313
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,7,0.008997333546479544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,15,0.009061333412925402
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,15,0.009189333145817121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,31,0.008992000172535578
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,31,0.00949866697192192
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,63,0.00901333304742972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,63,0.009061333412925402
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,127,0.009482666850090027
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,127,0.009125333279371262
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,255,0.009930666536092758
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,255,0.009109333157539368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,511,0.011050666371981302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,511,0.011141333729028702
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,16383,0.01741333305835724
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,1023,0.01097600037852923
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,2047,0.01081066702802976
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,2047,0.01109333336353302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,4095,0.01313599944114685
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,4095,0.013007999708255133
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,8191,0.015173333386580149
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,65535,0.036943999429543815
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,8191,0.015109332899252573
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,16383,0.01720533271630605
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,16383,0.017008000363906223
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,32767,0.033973333736260734
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,65535,0.05206400156021118
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,63,0.008853333070874214
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,65535,0.03570666660865148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,1,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,1,0.011055999745925268
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,3,0.008954666554927826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,3,0.009189333145817121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,7,0.009679999823371569
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,7,0.010757333288590113
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,15,0.009535999968647957
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,15,0.009568000212311745
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,3,0.009189333145817121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,31,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,31,0.010784000158309937
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,63,0.009152000149091085
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,63,0.009103999783595404
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,127,0.01091733326514562
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,127,0.011098666737476984
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,255,0.010901333143313726
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,255,0.010992000500361124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,511,0.01128000020980835
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,511,0.01128000020980835
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,1023,0.010847999403874079
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,1023,0.013189333180586496
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,1023,0.013056000073750814
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,2047,0.01516266663869222
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,2047,0.01505600040157636
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,32767,0.019359999646743137
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,8191,0.049584001302719116
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,4095,0.03145066648721695
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,8191,0.032773333291212715
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,4095,0.01878400022784869
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,1,0.01091733326514562
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,1,0.010874666273593903
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,3,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,7,0.011125333607196808
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,15,0.010944000134865442
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,7,0.010981333752473196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,31,0.010928000013033548
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,15,0.011061333119869232
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,31,0.009279999881982803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,63,0.010944000134865442
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,63,0.010874666273593903
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,127,0.010944000134865442
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,127,0.01099733387430509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,255,0.01101333275437355
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,255,0.0100853331387043
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,511,0.010981333752473196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,511,0.010911999891201654
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,1023,0.013199999928474426
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,1023,0.013125333935022354
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,4095,0.04782933493455251
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,4095,0.031082667410373688
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,2047,0.015322666615247726
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,8191,0.08480532964070638
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,1,0.011621333658695221
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,8191,0.04897599915663401
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,1,0.011002667248249054
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,3,0.011114666859308878
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,7,0.011519999553759893
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,3,0.011296000331640244
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,7,0.011296000331640244
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,15,0.01098666712641716
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,15,0.011141333729028702
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,31,0.011247999966144562
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,31,0.011247999966144562
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,63,0.01108266661564509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,63,0.011157333850860596
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,3,0.011114666859308878
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,127,0.01110400011142095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,127,0.01099733387430509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,255,0.01119999960064888
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,255,0.011061333119869232
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,2047,0.04797866443792979
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,511,0.013045333325862885
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,2047,0.03136000037193298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,1023,0.015119999647140503
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,1023,0.03029866764942805
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,4095,0.08477866649627686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,4095,0.04986133178075155
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,3,0.00966933307548364
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,2047,0.03014400104681651
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,8191,0.08714133501052856
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,7,0.009173333023985228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,3,0.009301333377758661
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,1,0.009269333134094873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,1,0.01080000028014183
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,8191,0.1581760048866272
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,15,0.011002667248249054
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,31,0.009328000247478485
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,31,0.008986666798591614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,63,0.009573333586255709
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,7,0.009077333534757296
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,127,0.009381333366036415
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,63,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,255,0.00914666677514712
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,255,0.011029332876205444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,511,0.011018666128317514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,511,0.011221333096424738
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,1023,0.011194666226704916
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,1023,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,2047,0.011071999867757162
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,2047,0.010890666395425797
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,4095,0.011098666737476984
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,4095,0.011045332998037338
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,8191,0.017263999829689663
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,8191,0.016794666647911072
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,16383,0.019440000255902607
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,16383,0.019285333653291065
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,32767,0.021418665846188862
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,32767,0.021221332252025604
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,65535,0.03770133356253306
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,65535,0.02351466566324234
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,1,0.008976000050703684
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,1,0.00903466654320558
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,3,0.009296000003814697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,3,0.00901333304742972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,7,0.009077333534757296
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,7,0.00897066667675972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,15,0.0099093330403169
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,31,0.00901333304742972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,31,0.00916800027092298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,511,0.0143306665122509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,63,0.009103999783595404
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,63,0.009072000160813332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,127,0.008954666554927826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,255,0.009077333534757296
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,255,0.009365333244204521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,127,0.009216000015536943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,511,0.01109333336353302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,511,0.011039999624093374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,1023,0.011050666371981302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,1023,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,2047,0.011130666981140772
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,2047,0.010879999647537867
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,4095,0.012901333471139273
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,8191,0.015146666516860327
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,15,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,127,0.01102399950226148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,16383,0.01738133281469345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,32767,0.03387200087308884
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,32767,0.01918399954835574
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,65535,0.0518453319867452
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,65535,0.035599999129772186
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,1,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,1,0.00916800027092298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,3,0.009008000294367472
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,3,0.009205333267649015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,7,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,7,0.008933333059151968
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,15,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,15,0.008938666433095932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,31,0.008901333436369896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,31,0.009279999881982803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,63,0.00922133338948091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,15,0.009173333023985228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,63,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,127,0.008938666433095932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,255,0.009216000015536943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,511,0.00926399976015091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,511,0.010954666882753372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,1023,0.009109333157539368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,1023,0.01033599985142549
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,2047,0.013007999708255133
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,2047,0.011930666863918304
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,4095,0.015082667271296183
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,4095,0.013274667163689932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,4095,0.01441066712141037
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,8191,0.017029333859682083
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,8191,0.015194666882356008
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,16383,0.03297599901755651
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,32767,0.03345600018898646
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,16383,0.01720000058412552
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,65535,0.08642133076985677
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,65535,0.0518506666024526
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,1,0.011029332876205444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,1,0.011039999624093374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,3,0.010949333508809408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,3,0.010954666882753372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,127,0.009194666519761086
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,7,0.010981333752473196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,15,0.010954666882753372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,15,0.01110400011142095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,31,0.01098666712641716
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,16383,0.018778666853904724
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,32767,0.04977599779764811
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,8191,0.01515199989080429
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,31,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,63,0.010911999891201654
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,63,0.011168000598748526
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,127,0.011039999624093374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,255,0.011125333607196808
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,511,0.012853333105643591
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,511,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,1023,0.015082667271296183
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,7,0.010847999403874079
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,255,0.009093333035707474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,1023,0.013189333180586496
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,2047,0.031343999008337654
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,2047,0.017050666113694508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,1,0.01099733387430509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,4095,0.04902400076389313
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,4095,0.03183999905983607
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,3,0.011125333607196808
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,3,0.01098666712641716
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,255,0.01101333275437355
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,127,0.010933333386977514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,7,0.012522666404644648
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,15,0.011727999895811081
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,15,0.010869332899649939
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,7,0.011247999966144562
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,31,0.011039999624093374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,63,0.011264000087976456
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,63,0.011402666568756104
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,127,0.01126933346192042
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,127,0.011114666859308878
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,255,0.012309333930412928
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,255,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,511,0.015157333264748255
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,511,0.013077333569526672
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,1023,0.02991466720898946
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,1023,0.015200000256299973
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,2047,0.049839998284975685
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,4095,0.0498986691236496
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,4095,0.0867199997107188
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,1,0.014906667172908783
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,2047,0.031397332747777305
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,31,0.011285333583752314
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,1,0.01312000056107839
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,3,0.013082666943470636
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,7,0.015002666662136713
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,7,0.013946666071812311
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,3,0.01509333277742068
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,15,0.013056000073750814
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,1,0.011157333850860596
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,15,0.014458666245142618
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,31,0.013056000073750814
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,31,0.014538666854302088
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,63,0.01509333277742068
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,63,0.01392000044385592
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,127,0.015029333531856537
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,127,0.012986666212479273
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,255,0.013280000537633896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,255,0.015237333873907724
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,511,0.030453334252039593
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,2047,0.0867146650950114
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,2047,0.050581331054369606
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,1023,0.04987733562787374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,1023,0.0325546662012736
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,4095,0.15871999661127725
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,4095,0.08758399883906047
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,1,0.013093333691358566
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,3,0.012949333836634954
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,1,0.012970666090647379
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,7,0.013007999708255133
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,15,0.01314666618903478
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,15,0.012986666212479273
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,31,0.012986666212479273
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,3,0.012960000584522883
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,7,0.013082666943470636
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,511,0.017210666090250015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,63,0.012928000340859095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,63,0.013056000073750814
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,127,0.012949333836634954
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,127,0.013007999708255133
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,255,0.012885333349307379
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,255,0.01322666679819425
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,511,0.015066667149464289
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,511,0.015082667271296183
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,1,0.015072000523408255
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,1,0.013338666409254074
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,1023,0.03169066707293192
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,1023,0.017370666066805523
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,31,0.01310933381319046
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,3,0.014906667172908783
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,3,0.013962666193644205
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,7,0.015237333873907724
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,7,0.01332266628742218
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,15,0.015168000012636185
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,31,0.013999999811251959
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,15,0.013370666652917862
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,63,0.015247999380032221
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,63,0.014074667046467463
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,31,0.015087999403476715
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,127,0.01504533365368843
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,127,0.013440000514189402
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,255,0.015013333410024643
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,255,0.013365333278973898
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,1,0.0194560003777345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,511,0.017152000218629837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,1,0.01738133281469345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,1023,0.03265066693226496
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,1023,0.04985066751639048
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,3,0.019333332777023315
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,3,0.017237332959969837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,7,0.019258666783571243
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,15,0.019098666807015736
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,31,0.017594666530688603
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,7,0.017173333714405697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,63,0.01926400015751521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,15,0.017456000049908955
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,31,0.019194666296243668
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,63,0.01729600007335345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,127,0.019461333751678467
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,511,0.03065066784620285
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,127,0.01729600007335345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,255,0.030479999879995983
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,511,0.04947733382383982
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,511,0.031770666440327965
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,1023,0.05193066596984863
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,1,0.015205333630243937
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,1023,0.08471999565760295
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,1,0.015141333142916361
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,3,0.01708799973130226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,7,0.01584533353646596
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,3,0.015210667004187902
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,15,0.015141333142916361
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,31,0.01543466622630755
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,31,0.015141333142916361
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,63,0.016885332763195038
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,7,0.01540800059835116
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,15,0.01613866661985715
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,63,0.015194666882356008
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,255,0.01802666609485944
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,127,0.015253332753976187
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,127,0.015002666662136713
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,255,0.015333333363135656
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,255,0.015146666516860327
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,511,0.01913600042462349
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,1,0.019296000401178997
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,511,0.031632001201311745
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,1,0.018138666947682697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,3,0.019461333751678467
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,3,0.017504000415404636
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,7,0.019215999792019527
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,7,0.0174346665541331
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,15,0.017344000438849132
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,15,0.019306667149066925
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,31,0.01747200017174085
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,31,0.0194560003777345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,63,0.017557332913080852
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,127,0.019199999670187633
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,127,0.017445333302021027
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,255,0.031498665610949196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,255,0.01743999992807706
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,511,0.04980800052483877
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,1,0.03147733211517334
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,511,0.03350399931271871
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,63,0.019285333653291065
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,1,0.027429332335789997
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,3,0.03038399914900462
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,7,0.03147733211517334
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,3,0.02759466568628947
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,15,0.031504000226656594
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,7,0.027786667148272198
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,31,0.03156800071398417
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,15,0.027952000498771667
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,31,0.02773333340883255
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,63,0.031504000226656594
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,1,0.009039999917149544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,1,0.009114666531483332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,63,0.027386667827765148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,3,0.009253333633144697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,127,0.027386667827765148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,7,0.009285333255926767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,127,0.03585600107908249
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,3,0.008943999807039896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,15,0.009365333244204521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,31,0.009061333412925402
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,31,0.009658666948477427
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,7,0.010757333288590113
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,63,0.009008000294367472
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,15,0.010442666709423065
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,63,0.009290666629870733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,255,0.051498666405677795
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,127,0.009434666484594345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,127,0.009258666386206945
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,255,0.009162666896979014
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,255,0.008965333302815756
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,511,0.01101333275437355
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,1023,0.011002667248249054
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,1023,0.011136000355084738
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,2047,0.01108266661564509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,255,0.0355679988861084
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,2047,0.011002667248249054
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,4095,0.013776000589132309
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,8191,0.016176000237464905
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,4095,0.014122666170199713
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,16383,0.01929066702723503
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,16383,0.017093333105246227
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,32767,0.03482666611671448
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,32767,0.021381333470344543
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,65535,0.05399466554323832
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,8191,0.016042667130629223
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,1,0.009050666665037474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,65535,0.03703466554482778
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,1,0.009578666960199675
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,511,0.053957333167394005
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,3,0.009039999917149544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,511,0.08864532907803853
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,3,0.009973333527644476
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,7,0.008912000184257826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,15,0.009205333267649015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,15,0.009328000247478485
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,31,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,63,0.008992000172535578
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,63,0.009258666386206945
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,127,0.00933333362142245
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,127,0.009216000015536943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,255,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,255,0.009408000235756239
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,511,0.011007999380429586
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,511,0.01110400011142095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,1023,0.011018666128317514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,1023,0.012149333953857422
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,4095,0.015103999525308609
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,2047,0.01313599944114685
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,31,0.009290666629870733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,4095,0.013093333691358566
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,8191,0.017136000096797943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,8191,0.015184000134468079
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,16383,0.032074667513370514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,16383,0.017759999881188076
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,32767,0.05156266689300537
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,32767,0.0335359995563825
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,65535,0.08706667025883992
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,65535,0.05217066903909048
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,1,0.009322666873534521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,1,0.009226666763424873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,3,0.009322666873534521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,511,0.010933333386977514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,7,0.009253333633144697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,7,0.010010666524370512
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,15,0.009306666751702627
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,15,0.009152000149091085
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,31,0.008938666433095932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,63,0.009162666896979014
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,63,0.009183999771873156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,127,0.009205333267649015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,7,0.009312000125646591
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,127,0.008949333180983862
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,255,0.009178666397929192
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,255,0.009392000113924345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,511,0.011007999380429586
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,1023,0.010954666882753372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,1023,0.011168000598748526
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,2047,0.013440000514189402
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,2047,0.012335999558369318
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,4095,0.01505600040157636
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,4095,0.013343999783198038
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,2047,0.011413333316644033
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,8191,0.03012799968322118
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,8191,0.015402667224407196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,16383,0.048453330993652344
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,16383,0.031632001201311745
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,32767,0.0499893327554067
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,32767,0.08444799979527791
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,65535,0.08672533432642619
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,65535,0.15637866655985513
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,3,0.009082666908701261
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,31,0.009328000247478485
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,511,0.011066666493813196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,1,0.023658665517965954
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,1,0.019365333020687103
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,3,0.023386667172114056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,3,0.019482667247454327
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,7,0.023215999205907185
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,7,0.01937066639463107
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,15,0.023247999449570973
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,15,0.0205226664741834
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,31,0.019354666272799175
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,63,0.019440000255902607
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,31,0.023386667172114056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,127,0.023285334308942158
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,127,0.019578666736682255
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,255,0.03143466760714849
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,255,0.021269333859284718
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,63,0.02348800003528595
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,1,0.027679999669392902
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,1,0.031504000226656594
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,3,0.031717332700888314
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,3,0.027461332579453785
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,7,0.03145066648721695
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,7,0.027808000644048054
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,15,0.03148799886306127
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,15,0.02739733209212621
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,31,0.0314026673634847
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,31,0.027642667293548584
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,63,0.02737066646416982
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,63,0.031445334355036415
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,127,0.03606933355331421
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,127,0.02940800040960312
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,255,0.0525546669960022
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,255,0.035536001125971474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,1,0.05663466453552246
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,1,0.04810666541258494
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,3,0.056645333766937256
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,3,0.04869333406289419
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,7,0.04894933104515076
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,7,0.05628266433874766
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,15,0.05644799768924713
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,15,0.04789866507053375
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,31,0.05738666653633118
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,31,0.048026666045188904
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,63,0.05764266848564148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,63,0.04794666667779287
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,127,0.061349332332611084
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,127,0.05233600238958994
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,255,0.062128002444903054
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,255,0.09286933143933614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,1,0.0377866675456365
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,1,0.031850665807724
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,3,0.03788800040880839
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,7,0.03772266705830892
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,7,0.03182933231194814
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,15,0.037802666425704956
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,15,0.03148799886306127
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,3,0.03230933348337809
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,31,0.037717332442601524
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,63,0.037605332831541695
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,31,0.03173333406448364
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,127,0.03965866565704346
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,63,0.0320266659061114
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,1,0.0545653353134791
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,127,0.033413333197434746
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,1,0.04779199759165446
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,3,0.04778666794300079
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,3,0.054858664671579994
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,7,0.05620799958705902
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,7,0.047557334105173744
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,15,0.055829331278800964
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,15,0.047210668524106346
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,31,0.055946667989095054
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,31,0.046096002062161766
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,63,0.05558399856090546
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,63,0.04665599763393402
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,127,0.06080000102519989
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,1,0.10143466790517171
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,127,0.05106133222579956
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,1,0.09277866284052531
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,3,0.10333333412806193
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,3,0.09297066926956177
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,7,0.10263466835021973
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,7,0.09308266639709473
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,15,0.09233599901199341
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,31,0.10179199775060017
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,15,0.10201600193977356
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,31,0.09288533528645833
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,1,0.009189333145817121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,3,0.01080000028014183
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,3,0.009178666397929192
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,7,0.010575999816258749
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,1,0.009290666629870733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,63,0.10284266869227092
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,7,0.009216000015536943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,15,0.009205333267649015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,31,0.009039999917149544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,31,0.009461333354314169
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,63,0.009578666960199675
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,127,0.009253333633144697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,127,0.009061333412925402
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,255,0.009173333023985228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,63,0.009279999881982803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,511,0.011039999624093374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,511,0.011215999722480774
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,1023,0.010949333508809408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,1023,0.01102399950226148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,2047,0.013306666165590286
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,4095,0.015077333897352219
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,4095,0.015200000256299973
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,8191,0.017445333302021027
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,8191,0.01739199956258138
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,16383,0.03350933392842611
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,16383,0.01987733319401741
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,32767,0.05215999980767568
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,32767,0.035749333600203194
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,1,0.009008000294367472
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,1,0.009130666653315226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,3,0.00897066667675972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,15,0.01091733326514562
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,3,0.00916800027092298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,7,0.00897066667675972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,7,0.009375999992092451
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,63,0.0915786623954773
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,15,0.009029333169261614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,15,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,31,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,255,0.010586666564146677
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,31,0.009152000149091085
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,63,0.009178666397929192
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,127,0.009338666374484697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,127,0.009648000200589498
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,255,0.009136000027259191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,2047,0.013002666334311167
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,511,0.011045332998037338
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,63,0.009152000149091085
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,1023,0.011498666057984034
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,1023,0.011018666128317514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,2047,0.012879999975363413
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,2047,0.01109333336353302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,4095,0.015109332899252573
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,4095,0.013237333546082178
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,8191,0.029861333469549816
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,8191,0.016575999557971954
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,127,0.11129599809646606
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,16383,0.048250665267308555
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,16383,0.031328000128269196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,32767,0.0848426620165507
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,32767,0.0498933345079422
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,3,0.00997866690158844
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,3,0.009039999917149544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,7,0.010106666634480158
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,7,0.009039999917149544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,127,0.09502933422724406
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,1,0.011018666128317514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,15,0.01009599988659223
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,15,0.009253333633144697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,31,0.009125333279371262
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,31,0.009082666908701261
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,63,0.009093333035707474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,255,0.009162666896979014
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,127,0.010954666882753372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,127,0.00955200009047985
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,511,0.01116266722480456
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,255,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,255,0.009008000294367472
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,511,0.011221333096424738
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,1023,0.01314666618903478
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,2047,0.0141546664138635
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,2047,0.013301332791646322
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,63,0.009098666409651438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,4095,0.015247999380032221
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,4095,0.030453334252039593
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,8191,0.048709332942962646
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,8191,0.03156800071398417
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,16383,0.0497920016447703
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,16383,0.08488532900810242
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,1,0.009434666484594345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,1,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,32767,0.15637333194414774
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,1,0.00927466650803884
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,3,0.009066666786869368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,32767,0.08700266480445862
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,3,0.009375999992092451
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,7,0.009050666665037474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,15,0.009183999771873156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,15,0.009637333452701569
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,7,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,31,0.00915733352303505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,31,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,63,0.00915733352303505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,63,0.009306666751702627
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,127,0.009477333476146063
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,127,0.00927466650803884
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,255,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,255,0.009808000177145004
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,511,0.010992000500361124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,511,0.011066666493813196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,1023,0.011226666470368704
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,1023,0.011039999624093374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,2047,0.012890666723251343
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,2047,0.012181332955757776
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,4095,0.015274666249752045
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,4095,0.013264000415802002
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,8191,0.01714666684468587
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,16383,0.031343999008337654
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,16383,0.04970666766166687
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,511,0.011002667248249054
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,1,0.009279999881982803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,1023,0.010981333752473196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,32767,0.08695466319719951
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,32767,0.0505920002857844
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,3,0.009103999783595404
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,3,0.0106133334338665
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,7,0.00891733355820179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,7,0.009226666763424873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,15,0.00943999985853831
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,15,0.009706666693091393
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,31,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,31,0.009317333499590555
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,63,0.009194666519761086
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,63,0.008922666932145754
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,127,0.009343999748428663
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,127,0.009824000298976898
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,255,0.009103999783595404
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,511,0.010944000134865442
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,511,0.010933333386977514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,1023,0.01268799975514412
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,1023,0.01129066695769628
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,2047,0.01302933320403099
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,4095,0.02976000060637792
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,4095,0.015370666980743408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,8191,0.04825599988301595
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,8191,0.031744000812371574
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,16383,0.08530666430791219
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,16383,0.050186668833096824
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,32767,0.15963199734687805
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,1,0.010960000256697336
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,1,0.009535999968647957
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,32767,0.08695466319719951
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,8191,0.03051200012365977
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,3,0.009173333023985228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,3,0.011141333729028702
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,7,0.010650667051474253
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,7,0.010037333394090334
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,1,0.009253333633144697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,15,0.011007999380429586
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,15,0.009583999713261923
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,31,0.010224000240365664
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,31,0.009797333429257074
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,63,0.011066666493813196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,63,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,127,0.010666667173306147
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,255,0.01081066702802976
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,255,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,255,0.010661333799362183
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,511,0.011152000476916632
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,511,0.011002667248249054
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,1023,0.013151999562978745
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,2047,0.029370665550231934
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,1023,0.013082666943470636
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,2047,0.015109332899252573
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,4095,0.04785066843032837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,8191,0.08435733119646709
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,4095,0.030154667794704437
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,8191,0.04861866434415182
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,16383,0.08595732847849528
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,16383,0.15679466724395752
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,1,0.011114666859308878
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,1,0.011306667079528173
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,32767,0.16024000446001688
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,32767,0.301199992497762
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,3,0.011120000233252844
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,7,0.011760000139474869
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,15,0.011045332998037338
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,15,0.01137599969903628
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,127,0.010186666622757912
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,31,0.011695999652147293
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,63,0.010992000500361124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,127,0.011029332876205444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,127,0.011978667229413986
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,2047,0.013338666409254074
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,255,0.011029332876205444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,255,0.01118933285276095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,511,0.014954666296641031
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,511,0.013248000293970108
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,1023,0.029631999631722767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,1023,0.015930666277805965
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,4095,0.08473599950472514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,4095,0.04990399877230326
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,2047,0.0481279989083608
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,3,0.01110400011142095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,8191,0.08685333530108134
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,8191,0.15744533141454062
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,7,0.011157333850860596
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,16383,0.30373332897822064
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,1,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,31,0.011109333485364914
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,16383,0.15989866852760315
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,1,0.008997333546479544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,3,0.009039999917149544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,63,0.011098666737476984
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,3,0.00891733355820179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,7,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,15,0.009082666908701261
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,31,0.00903466654320558
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,15,0.00973866693675518
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,31,0.009061333412925402
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,63,0.008965333302815756
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,32767,0.6002293427785238
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,63,0.009082666908701261
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,127,0.009354666496316591
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,127,0.009119999905427298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,255,0.008976000050703684
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,511,0.009621333330869675
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,511,0.01116266722480456
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,1023,0.009130666653315226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,255,0.00895999992887179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,2047,0.009717333440979322
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,2047,0.010981333752473196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,4095,0.011136000355084738
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,4095,0.011087999989589056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,8191,0.015024000157912573
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,8191,0.01504533365368843
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,16383,0.019146667172511418
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,16383,0.01775466650724411
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,32767,0.027285332481066387
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,32767,0.027509334186712902
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,65535,0.031583999594052635
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,65535,0.029919999341169994
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,131071,0.032245332996050514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,131071,0.03186133255561193
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,1,0.009002666920423508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,2047,0.03130666663249334
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,3,0.008938666433095932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,3,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,7,0.009119999905427298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,15,0.008997333546479544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,7,0.009359999870260557
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,15,0.009269333134094873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,31,0.00892800030608972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,31,0.009130666653315226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,63,0.008954666554927826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,63,0.009088000282645226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,127,0.00927466650803884
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,255,0.008933333059151968
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,255,0.00916800027092298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,511,0.010026666646202406
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,511,0.009189333145817121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,1023,0.009050666665037474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,7,0.009226666763424873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,1023,0.009306666751702627
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,2047,0.009114666531483332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,2047,0.00914666677514712
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,4095,0.011055999745925268
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,4095,0.011071999867757162
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,8191,0.014965333044528961
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,8191,0.01509333277742068
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,16383,0.019296000401178997
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,16383,0.018730666488409042
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,32767,0.02276800076166789
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,32767,0.021407999098300934
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,65535,0.023962666591008503
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,65535,0.023306667804718018
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,131071,0.04186666508515676
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,131071,0.02533866713444392
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,1,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,1,0.009189333145817121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,3,0.008933333059151968
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,3,0.009050666665037474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,7,0.009375999992092451
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,1,0.00903466654320558
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,7,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,15,0.008885333314538002
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,31,0.009770666559537252
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,63,0.009056000038981438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,63,0.00916800027092298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,127,0.009349333122372627
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,127,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,255,0.00890666681031386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,511,0.011029332876205444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,511,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,1023,0.009317333499590555
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,1023,0.008976000050703684
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,2047,0.010992000500361124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,2047,0.011007999380429586
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,32767,0.30727465947469074
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,4095,0.010874666273593903
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,4095,0.011152000476916632
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,8191,0.015002666662136713
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,8191,0.015157333264748255
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,1023,0.009029333169261614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,16383,0.017397332936525345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,16383,0.01711999997496605
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,32767,0.021226666867733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,32767,0.019280000279347103
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,65535,0.03659199923276901
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,65535,0.02124800036350886
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,131071,0.05541333556175232
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,131071,0.03774933268626531
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,1,0.00955200009047985
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,1,0.008874666566650072
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,3,0.009039999917149544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,15,0.008943999807039896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,3,0.009216000015536943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,7,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,7,0.009589333087205887
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,31,0.009258666386206945
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,15,0.008954666554927826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,15,0.009141333401203156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,31,0.009568000212311745
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,63,0.009952000031868616
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,127,0.009269333134094873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,127,0.009088000282645226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,127,0.00922133338948091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,255,0.010373333469033241
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,255,0.01009599988659223
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,255,0.00984533317387104
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,511,0.010954666882753372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,511,0.009216000015536943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,1023,0.010538666198650995
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,1023,0.010885333021481832
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,2047,0.01118933285276095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,4095,0.013167999684810638
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,8191,0.015290666371583939
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,16383,0.017344000438849132
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,16383,0.01709866647919019
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,32767,0.033439998825391136
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,32767,0.01941866676012675
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,65535,0.05204799771308899
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,65535,0.03521066655715307
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,131071,0.08815466364224751
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,1,0.008890666688481966
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,131071,0.05439466734727224
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,3,0.008879999940594038
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,3,0.00903466654320558
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,7,0.008832000195980072
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,7,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,15,0.008896000062425932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,15,0.008863999818762144
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,31,0.008933333059151968
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,31,0.009077333534757296
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,31,0.009141333401203156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,63,0.009253333633144697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,63,0.008938666433095932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,127,0.009093333035707474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,127,0.009109333157539368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,255,0.008997333546479544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,2047,0.010847999403874079
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,255,0.00892800030608972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,511,0.00997866690158844
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,8191,0.015061333775520325
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,1023,0.008965333302815756
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,1023,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,2047,0.01097600037852923
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,2047,0.010933333386977514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,4095,0.010960000256697336
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,4095,0.010911999891201654
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,8191,0.015050667027632395
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,8191,0.01492799942692121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,16383,0.019461333751678467
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,16383,0.01912533367673556
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,32767,0.023408000667889912
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,32767,0.02145066608985265
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,65535,0.023455999791622162
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,65535,0.023247999449570973
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,131071,0.03994133323431015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,1,0.009701333319147428
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,1,0.00895999992887179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,1,0.00898133342464765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,3,0.009039999917149544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,7,0.009178666397929192
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,7,0.00895999992887179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,15,0.009205333267649015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,31,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,63,0.009018666421373686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,31,0.009130666653315226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,63,0.008869333192706108
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,63,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,127,0.008853333070874214
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,127,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,255,0.008901333436369896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,4095,0.013082666943470636
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,511,0.010698666175206503
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,255,0.009898666913310686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,511,0.009359999870260557
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,511,0.010949333508809408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,1023,0.009050666665037474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,1023,0.010858666151762009
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,2047,0.011045332998037338
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,2047,0.010960000256697336
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,4095,0.010853332777818045
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,4095,0.01091733326514562
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,8191,0.015392000476519266
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,8191,0.015487999965747198
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,16383,0.017637333522240322
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,16383,0.017210666090250015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,32767,0.02111999938885371
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,32767,0.019167999426523846
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,65535,0.03709866603215536
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,65535,0.021397332350413006
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,131071,0.05585599939028422
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,131071,0.038831998904546104
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,1,0.008933333059151968
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,1,0.009370666618148485
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,131071,0.025386666258176167
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,3,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,7,0.00898133342464765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,15,0.00895999992887179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,3,0.009258666386206945
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,31,0.009018666421373686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,31,0.009269333134094873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,63,0.009029333169261614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,63,0.009296000003814697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,127,0.009242666885256767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,127,0.009045333291093508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,255,0.009242666885256767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,15,0.009183999771873156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,255,0.009061333412925402
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,511,0.010981333752473196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,1023,0.009317333499590555
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,1023,0.010421333213647207
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,2047,0.011071999867757162
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,2047,0.010949333508809408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,4095,0.013125333935022354
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,4095,0.01333333303531011
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,8191,0.015189333508412043
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,8191,0.015077333897352219
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,16383,0.017125333348910015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,16383,0.017263999829689663
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,32767,0.03342933456103007
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,32767,0.019237333287795384
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,65535,0.05197333296140035
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,65535,0.03570133447647095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,131071,0.08740267157554626
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,1,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,1,0.009045333291093508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,3,0.009258666386206945
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,7,0.010112000008424124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,7,0.010122666756312052
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,3,0.009039999917149544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,7,0.008890666688481966
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,15,0.009152000149091085
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,15,0.009136000027259191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,15,0.009962666779756546
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,31,0.00890666681031386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,31,0.009423999736706415
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,63,0.009103999783595404
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,63,0.009103999783595404
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,127,0.00966933307548364
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,127,0.009088000282645226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,511,0.009925333162148794
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,255,0.008896000062425932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,255,0.00927466650803884
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,511,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,511,0.010101333260536194
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,1023,0.011061333119869232
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,1023,0.009349333122372627
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,2047,0.012778667112191519
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,4095,0.015146666516860327
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,4095,0.013104000439246496
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,8191,0.017077332983414333
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,8191,0.01525866612792015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,16383,0.018464000274737675
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,16383,0.03173333406448364
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,131071,0.05530133346716563
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,32767,0.03339199970165888
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,32767,0.05072000126043955
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,65535,0.051669334371884666
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,65535,0.08641599615414937
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,3,0.009039999917149544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,131071,0.08914132912953694
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,131071,0.1583626667658488
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,1,0.009205333267649015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,1,0.011141333729028702
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,3,0.009226666763424873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,3,0.011066666493813196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,7,0.009136000027259191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,15,0.00916800027092298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,7,0.009296000003814697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,15,0.010832000523805618
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,31,0.009029333169261614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,31,0.010981333752473196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,63,0.008933333059151968
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,63,0.009312000125646591
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,127,0.009573333586255709
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,255,0.011125333607196808
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,255,0.009183999771873156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,511,0.01101333275437355
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,511,0.01099733387430509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,1023,0.012533333152532578
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,1023,0.01126933346192042
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,2047,0.013343999783198038
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,2047,0.012906666845083237
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,2047,0.013072000195582708
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,4095,0.030896000564098358
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,4095,0.015184000134468079
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,8191,0.0489279975493749
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,8191,0.0317493329445521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,1,0.010992000500361124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,16383,0.049738665421803795
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,3,0.01121066634853681
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,16383,0.0872320036093394
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,7,0.010981333752473196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,1,0.011077333241701126
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,3,0.011029332876205444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,15,0.010629333555698395
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,15,0.011050666371981302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,31,0.010837333897749582
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,31,0.010928000013033548
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,63,0.01080000028014183
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,63,0.009279999881982803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,127,0.011055999745925268
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,127,0.011381333072980246
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,255,0.011066666493813196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,255,0.010933333386977514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,511,0.011264000087976456
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,511,0.011183999478816986
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,127,0.00901333304742972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,1023,0.01302933320403099
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,1023,0.013248000293970108
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,4095,0.04798933366934458
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,4095,0.03140799949566523
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,2047,0.0295413335164388
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,8191,0.08450133601824443
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,8191,0.04987733562787374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,1,0.011322667201360067
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,16383,0.08704533179601033
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,16383,0.15848533312479654
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,3,0.011178666104873022
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,3,0.010879999647537867
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,1,0.011114666859308878
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,7,0.011658667276302973
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,7,0.010954666882753372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,7,0.010928000013033548
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,15,0.01267733300725619
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,31,0.01098666712641716
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,63,0.011584000041087469
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,63,0.011098666737476984
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,127,0.011551999797423681
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,255,0.01102399950226148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,255,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,511,0.01440000037352244
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,511,0.01293333371480306
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,1023,0.029365333418051403
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,2047,0.01533866673707962
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,1023,0.015184000134468079
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,2047,0.04782933493455251
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,2047,0.031658666829268135
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,4095,0.08545066912968953
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,8191,0.15768532951672873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,4095,0.050010666251182556
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,8191,0.08768533666928609
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,15,0.01102399950226148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,31,0.011178666104873022
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,1,0.013114667187134424
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,1,0.014373333503802618
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,3,0.01523200049996376
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,3,0.013306666165590286
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,16383,0.1628159979979197
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,7,0.01349866638580958
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,7,0.013290667285521826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,15,0.014997333288192749
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,15,0.013104000439246496
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,31,0.015072000523408255
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,31,0.013258667041858038
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,63,0.014901333798964819
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,63,0.01328533391157786
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,127,0.013141332815090815
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,127,0.015210667004187902
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,255,0.014826666563749313
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,255,0.01303999995191892
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,511,0.02956266701221466
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,511,0.01722666621208191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,1023,0.04775466521581014
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,2047,0.08506666620572408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,2047,0.04987733562787374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,127,0.011157333850860596
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,8191,0.30456533034642536
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,16383,0.3033493359883626
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,4095,0.08621866504351298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,4095,0.15821866194407144
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,1,0.009077333534757296
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,8191,0.1591093341509501
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,1,0.00895999992887179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,3,0.009082666908701261
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,7,0.009088000282645226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,3,0.009290666629870733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,15,0.008943999807039896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,7,0.008949333180983862
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,15,0.009103999783595404
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,31,0.009375999992092451
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,31,0.008986666798591614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,63,0.008922666932145754
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,127,0.009205333267649015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,127,0.009930666536092758
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,255,0.00914666677514712
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,16383,0.5976159969965616
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,255,0.009183999771873156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,511,0.009648000200589498
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,511,0.01102399950226148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,1023,0.009338666374484697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,2047,0.009370666618148485
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,2047,0.011050666371981302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,4095,0.011146667102972666
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,4095,0.011178666104873022
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,8191,0.015317333241303762
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,8191,0.015173333386580149
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,16383,0.018090666582187016
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,16383,0.30352532863616943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,32767,0.020181333025296528
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,1023,0.00956266683836778
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,1023,0.031856000423431396
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,65535,0.03551466763019562
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,131071,0.05421866476535797
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,1,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,131071,0.03746666759252548
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,65535,0.02128533273935318
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,3,0.009317333499590555
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,7,0.010234666367371878
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,7,0.009008000294367472
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,15,0.009183999771873156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,15,0.00897066667675972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,31,0.009082666908701261
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,31,0.009189333145817121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,63,0.009178666397929192
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,63,0.009056000038981438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,127,0.008954666554927826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,127,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,255,0.008976000050703684
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,255,0.009077333534757296
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,511,0.010960000256697336
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,511,0.01081066702802976
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,1023,0.009258666386206945
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,1023,0.009056000038981438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,2047,0.011055999745925268
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,2047,0.01098666712641716
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,4095,0.013141332815090815
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,63,0.008869333192706108
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,4095,0.013199999928474426
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,8191,0.015061333775520325
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,8191,0.015098666151364645
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,16383,0.01730666682124138
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,16383,0.015482666591803232
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,32767,0.03352533280849457
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,32767,0.019152000546455383
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,65535,0.05234666665395101
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,65535,0.035461333890755974
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,16383,0.01735466718673706
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,32767,0.019226666539907455
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,131071,0.05382933219273885
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,131071,0.08850133419036865
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,1,0.008885333314538002
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,3,0.009018666421373686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,1,0.009045333291093508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,3,0.00916800027092298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,7,0.009152000149091085
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,7,0.009029333169261614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,3,0.00891733355820179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,15,0.010128000130256018
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,31,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,31,0.009242666885256767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,63,0.009082666908701261
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,63,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,127,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,127,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,255,0.00898133342464765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,255,0.009002666920423508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,511,0.01099733387430509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,511,0.01098666712641716
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,1023,0.009077333534757296
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,2047,0.013034666577974955
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,2047,0.012890666723251343
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,4095,0.01504533365368843
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,4095,0.013077333569526672
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,8191,0.015087999403476715
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,8191,0.017173333714405697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,16383,0.032261334359645844
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,16383,0.01721599946419398
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,32767,0.033514666060606636
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,32767,0.05019199848175049
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,65535,0.08630399902661641
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,65535,0.052298665046691895
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,1,0.008986666798591614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,131071,0.08868267138799031
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,131071,0.1581493318080902
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,3,0.009402666861812273
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,7,0.009290666629870733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,7,0.009797333429257074
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,1,0.009375999992092451
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,15,0.009162666896979014
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,31,0.009743999689817429
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,31,0.009029333169261614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,15,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,63,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,63,0.009413333609700203
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,127,0.008943999807039896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,127,0.009717333440979322
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,255,0.009472000102202097
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,255,0.009269333134094873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,1023,0.010965333630641302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,511,0.011173332730929056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,511,0.009839999799927076
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,1023,0.010960000256697336
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,1023,0.011258666714032492
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,2047,0.011370666325092316
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,2047,0.0129120002190272
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,4095,0.01498666654030482
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,4095,0.013264000415802002
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,8191,0.029482667644818623
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,8191,0.01540800059835116
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,16383,0.031717332700888314
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,16383,0.0481279989083608
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,32767,0.08458133538564046
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,32767,0.04994666576385498
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,65535,0.1560640037059784
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,1,0.009018666421373686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,65535,0.08695466319719951
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,131071,0.3021226723988851
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,3,0.009136000027259191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,131071,0.16113066673278809
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,1,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,3,0.011045332998037338
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,1,0.010858666151762009
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,15,0.009119999905427298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,3,0.01118933285276095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,7,0.011007999380429586
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,7,0.010911999891201654
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,15,0.010816000401973724
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,15,0.011061333119869232
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,31,0.011029332876205444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,31,0.01102399950226148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,63,0.010960000256697336
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,63,0.01145600030819575
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,127,0.011109333485364914
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,127,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,255,0.011045332998037338
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,255,0.010826667149861654
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,511,0.011066666493813196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,511,0.011109333485364914
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,1023,0.013157332936922709
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,1023,0.013061333447694778
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,2047,0.030826665461063385
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,2047,0.015130666395028433
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,4095,0.04849599798520406
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,4095,0.03145066648721695
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,8191,0.04984533290068308
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,1,0.011322667201360067
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,3,0.01129066695769628
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,7,0.011333333949247995
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,3,0.011018666128317514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,15,0.011946666985750198
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,7,0.0120319997270902
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,15,0.01108266661564509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,31,0.011231999844312668
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,31,0.01101333275437355
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,63,0.011296000331640244
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,63,0.011066666493813196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,127,0.012042666474978128
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,127,0.010992000500361124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,255,0.011007999380429586
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,255,0.011359999577204386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,511,0.013679999858140945
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,511,0.013002666334311167
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,8191,0.08679466446240743
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,1,0.01257066677014033
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,1023,0.029648000995318096
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,1023,0.01532799998919169
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,4095,0.05045333504676819
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,4095,0.08621333042780559
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,8191,0.15967466433842978
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,2047,0.04852266609668732
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,2047,0.031514666974544525
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,8191,0.0888266662756602
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,1,0.01498666654030482
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,1,0.01303999995191892
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,3,0.015135999768972397
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,7,0.015205333630243937
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,15,0.013642666240533194
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,31,0.015087999403476715
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,31,0.013301332791646322
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,63,0.015119999647140503
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,7,0.013365333278973898
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,15,0.01522133375207583
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,63,0.013274667163689932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,127,0.013765333841244379
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,127,0.015541333705186844
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,255,0.013125333935022354
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,255,0.015077333897352219
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,511,0.017130666722853977
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,511,0.03011200080315272
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,2047,0.08629866441090901
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,2047,0.05050666630268097
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,1023,0.04795200129350027
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,4095,0.15981333454449972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,3,0.01313599944114685
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,1,0.019253333409627277
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,4095,0.08822932839393616
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,3,0.019253333409627277
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,8191,0.16115199526151022
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,8191,0.30849067370096844
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,3,0.017210666090250015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,7,0.019359999646743137
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,1,0.01736533393462499
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,15,0.019509332875410717
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,15,0.017349333812793095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,31,0.01729600007335345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,63,0.0191040001809597
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,63,0.017152000218629837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,127,0.019146667172511418
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,1023,0.03203733265399933
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,127,0.01747200017174085
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,255,0.01706133286158244
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,255,0.029493334392706554
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,511,0.047930667797724404
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,511,0.03142933299144109
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,1023,0.050240000089009605
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,7,0.017130666722853977
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,1023,0.08475200335184734
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,2047,0.08753599723180135
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,2047,0.15705600380897522
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,31,0.019178666174411774
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,1,0.011434666812419891
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,4095,0.29923733075459796
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,4095,0.16074666380882263
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,1,0.011077333241701126
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,3,0.012709333250919977
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,3,0.01109333336353302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,8191,0.30904533465703327
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,7,0.011312000453472137
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,8191,0.5913706620534261
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,15,0.011584000041087469
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,7,0.011152000476916632
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,15,0.01121066634853681
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,31,0.012357333054145178
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,31,0.011130666981140772
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,63,0.011258666714032492
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,63,0.011039999624093374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,127,0.011221333096424738
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,255,0.011834666132926941
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,255,0.011264000087976456
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,511,0.015013333410024643
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,511,0.012960000584522883
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,1023,0.02996266633272171
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,1,0.015008000036080679
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,1,0.013397333522637686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,2047,0.031530665854612984
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,3,0.015008000036080679
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,3,0.013280000537633896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,7,0.015087999403476715
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,127,0.011296000331640244
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,7,0.013173333058754602
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,15,0.01481066644191742
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,15,0.013061333447694778
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,31,0.015114666273196539
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,1023,0.01605333387851715
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,31,0.013248000293970108
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,63,0.01301866645614306
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,63,0.015130666395028433
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,2047,0.04894400139649709
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,127,0.015066667149464289
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,127,0.014015999933083853
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,255,0.01525866612792015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,1023,0.049600000182787575
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,255,0.013381333400805792
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,511,0.030378667016824085
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,511,0.01710933322707812
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,1023,0.0325546662012736
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,1,0.01921066641807556
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,2047,0.049925332268079124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,2047,0.08700266480445862
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,3,0.019402666638294857
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,3,0.01710933322707812
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,7,0.019359999646743137
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,15,0.019498666127522785
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,7,0.01747200017174085
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,31,0.01724799970785777
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,63,0.019173332800467808
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,127,0.019354666272799175
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,127,0.01756799966096878
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,31,0.019141333798567455
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,15,0.01714666684468587
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,255,0.03052799900372823
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,1,0.01721599946419398
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,511,0.04889066517353058
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,511,0.03140799949566523
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,63,0.017877332866191864
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,2047,0.08930133779843648
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,2047,0.15811199943224588
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,1023,0.08461333314577739
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,1023,0.05123733480771383
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,1,0.02942399928967158
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,1,0.025765334566434223
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,3,0.02959999938805898
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,255,0.017114666601022083
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,3,0.025455998877684276
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,7,0.02978666623433431
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,15,0.025519999365011852
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,31,0.029487999776999157
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,31,0.025786665578683216
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,63,0.025381334125995636
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,63,0.02938666691382726
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,127,0.033600000043710075
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,127,0.026709333062171936
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,7,0.02571200082699458
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,15,0.02942399928967158
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,255,0.04961066444714864
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,255,0.033520000676314034
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,1023,0.08663466572761536
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,1023,0.1565546691417694
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,2047,0.30004266897837323
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,2047,0.15851199626922607
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,511,0.05086933573087057
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,511,0.08462400237719218
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,1,0.015173333386580149
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,3,0.015087999403476715
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,1,0.013130666067202887
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,7,0.015157333264748255
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,7,0.013514666507641474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,3,0.013445333888133367
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,31,0.015168000012636185
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,15,0.014975999792416891
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,15,0.013162666310866674
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,31,0.013130666067202887
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,63,0.014906667172908783
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,63,0.013077333569526672
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,127,0.015216000378131866
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,127,0.013295999417702356
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,255,0.013232000172138214
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,255,0.01505600040157636
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,511,0.01714666684468587
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,511,0.030458666384220123
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,1,0.017338667064905167
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,1023,0.04970666766166687
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,3,0.01921066641807556
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,3,0.017370666066805523
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,7,0.019461333751678467
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,7,0.018522666146357853
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,15,0.019679999599854153
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,1,0.019274666905403137
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,15,0.01741333305835724
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,1023,0.03194666653871536
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,31,0.01940800001223882
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,63,0.02773866554101308
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,63,0.0173333336909612
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,127,0.019173332800467808
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,127,0.017162666966517765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,255,0.030746666093667347
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,255,0.017344000438849132
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,31,0.01714666684468587
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,511,0.04951466619968414
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,511,0.033589333295822144
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,1,0.031504000226656594
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,1,0.027445333699385326
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,1023,0.08735466996828715
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,1023,0.050704002380371094
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,3,0.031530665854612984
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,3,0.02762666592995326
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,7,0.03146133323510488
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,15,0.03141866624355316
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,15,0.02737066646416982
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,63,0.03145600110292435
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,7,0.02794133375088374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,63,0.027647999425729115
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,127,0.035536001125971474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,127,0.028768000503381092
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,31,0.031530665854612984
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,255,0.0513919989267985
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,255,0.035562666753927864
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,31,0.027456000447273254
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,511,0.05330666899681091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,1,0.05253866811593374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,511,0.08834133545557658
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,1,0.04391466577847799
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,3,0.04417600234349569
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,3,0.05221866567929586
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,1023,0.1602079967657725
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,1023,0.09110933542251587
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,7,0.043968002001444496
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,15,0.043824002146720886
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,31,0.04403733213742574
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,31,0.05402666827042898
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,7,0.05273599922657013
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,127,0.05635199944178263
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,15,0.054458667834599815
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,127,0.04786133269468943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,255,0.08797333637873332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,255,0.05615466833114624
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,1023,0.30245866378148395
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,3,0.009130666653315226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,63,0.05385066568851471
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,3,0.009258666386206945
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,7,0.009082666908701261
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,15,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,15,0.009125333279371262
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,7,0.008879999940594038
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,1023,0.16268266240755716
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,31,0.008986666798591614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,31,0.008986666798591614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,63,0.009194666519761086
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,127,0.009178666397929192
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,511,0.16010133425394693
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,127,0.009109333157539368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,255,0.008869333192706108
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,255,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,511,0.008954666554927826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,1023,0.009413333609700203
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,1023,0.00996800015370051
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,2047,0.010837333897749582
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,2047,0.010869332899649939
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,4095,0.013183999806642532
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,4095,0.012944000462690989
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,8191,0.015301333119471868
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,63,0.04427733520666758
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,16383,0.017445333302021027
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,511,0.011039999624093374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,16383,0.016837333639462788
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,32767,0.0194560003777345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,65535,0.05202133456865946
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,65535,0.03515733281771342
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,131071,0.08874666690826416
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,131071,0.05288533369700114
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,511,0.08948799967765808
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,1,0.009338666374484697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,1,0.00915733352303505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,1,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,63,0.009359999870260557
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,1,0.009349333122372627
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,3,0.00916800027092298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,7,0.009141333401203156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,7,0.00895999992887179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,15,0.009002666920423508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,15,0.009461333354314169
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,31,0.008912000184257826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,31,0.009301333377758661
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,63,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,63,0.009328000247478485
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,127,0.009141333401203156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,127,0.00897066667675972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,255,0.008992000172535578
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,255,0.009173333023985228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,511,0.00973866693675518
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,511,0.01099733387430509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,1023,0.010970667004585266
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,1023,0.00949866697192192
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,2047,0.012165332833925882
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,4095,0.015066667149464289
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,4095,0.013066666821638743
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,8191,0.017071999609470367
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,8191,0.015114666273196539
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,8191,0.015146666516860327
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,16383,0.0324799989660581
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,16383,0.018757333358128864
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,32767,0.05091733237107595
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,32767,0.03340800106525421
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,32767,0.03365866591533025
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,65535,0.05301333467165629
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,1,0.009216000015536943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,131071,0.09134933352470398
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,131071,0.16009066502253214
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,1,0.009375999992092451
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,3,0.010128000130256018
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,3,0.01341333364446958
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,7,0.00898133342464765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,7,0.009754666437705358
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,15,0.009919999788204828
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,31,0.00922133338948091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,31,0.009514666472872099
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,63,0.00927466650803884
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,63,0.009290666629870733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,127,0.009813333551088968
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,127,0.009183999771873156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,255,0.009312000125646591
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,255,0.009306666751702627
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,511,0.011007999380429586
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,511,0.010954666882753372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,1023,0.011253333340088526
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,2047,0.013013333082199097
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,1023,0.010965333630641302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,2047,0.012842666357755661
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,2047,0.011061333119869232
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,4095,0.015168000012636185
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,4095,0.013013333082199097
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,8191,0.029663999875386555
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,8191,0.017103999853134155
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,3,0.009375999992092451
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,16383,0.04790399968624115
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,16383,0.03159466634194056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,32767,0.08473066488901775
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,32767,0.049914668003718056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,65535,0.08814400434494019
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,65535,0.08708799878756206
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,65535,0.1574613352616628
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,1,0.008954666554927826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,1,0.009722666814923286
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,3,0.009312000125646591
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,131071,0.16084800163904825
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,3,0.009290666629870733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,131071,0.30115199089050293
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,15,0.00914666677514712
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,15,0.009279999881982803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,31,0.009189333145817121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,15,0.009072000160813332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,63,0.010944000134865442
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,31,0.009125333279371262
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,63,0.009002666920423508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,127,0.009285333255926767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,127,0.00960533320903778
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,255,0.010949333508809408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,511,0.01102399950226148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,255,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,511,0.01090666651725769
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,1023,0.013002666334311167
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,1023,0.011338666081428528
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,2047,0.014111999422311783
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,2047,0.013104000439246496
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,4095,0.015279999623696009
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,8191,0.04816000163555145
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,8191,0.03145600110292435
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,16383,0.04956266780694326
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,16383,0.08444266517957051
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,32767,0.08499733606974284
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,65535,0.3003946741422017
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,65535,0.1572213371594747
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,131071,0.5881386597951254
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,7,0.009205333267649015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,7,0.008943999807039896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,4095,0.029616000751654308
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,32767,0.15641599893569946
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,131071,0.3023786743481954
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,1,0.019413333386182785
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,1,0.01759999990463257
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,3,0.01773333301146825
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,7,0.01918399954835574
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,3,0.019258666783571243
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,7,0.01741333305835724
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,15,0.020319999506076176
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,15,0.017498667041460674
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,31,0.01951466624935468
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,31,0.017221332838137943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,63,0.01937599976857503
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,63,0.016970666746298473
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,127,0.019509332875410717
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,127,0.01798933371901512
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,255,0.0315733328461647
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,255,0.01793066660563151
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,1,0.03145600110292435
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,511,0.05004266897837321
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,1,0.027471999327341717
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,511,0.03254399945338567
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,3,0.02762666592995326
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,3,0.03169599920511246
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,7,0.03158933420976003
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,7,0.027664000789324444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,15,0.031471999982992806
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,31,0.031471999982992806
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,31,0.02739199995994568
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,15,0.027493332823117573
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,63,0.03149333347876867
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,63,0.027653334041436512
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,127,0.03566399961709976
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,127,0.028037334481875103
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,255,0.051882664362589516
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,255,0.0360959991812706
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,511,0.08874666690826416
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,1,0.05630933245023092
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,511,0.053898667295773826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,1,0.048698668678601585
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,3,0.058176000912984215
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,3,0.04781866570313772
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,7,0.05635733405749003
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,7,0.047877331574757896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,15,0.04784533381462097
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,15,0.058005332946777344
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,31,0.04809600114822388
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,31,0.05682666599750519
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,63,0.057855998476346336
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,63,0.04786666731039683
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,127,0.06004266440868378
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,127,0.05226133267084757
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,255,0.09287466605504353
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,1,0.09532800316810608
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,1,0.08363200227419536
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,255,0.06178666651248932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,511,0.09708799918492635
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,511,0.16487466295560202
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,3,0.09547199805577596
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,3,0.08474666873613994
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,7,0.09526399771372478
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,7,0.08342933654785156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,15,0.09528000156084697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,15,0.0844586690266927
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,63,0.0959999958674113
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,63,0.08473066488901775
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,127,0.1011786659558614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,127,0.08585066596666972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,31,0.08381866415341695
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,31,0.09530133008956909
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,255,0.16459199786186218
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,255,0.10244799653689067
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,511,0.16867733001708984
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,1,0.03154666721820831
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,511,0.30800533294677734
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,1,0.027674667537212372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,3,0.02754133443037669
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,3,0.0316746657093366
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,7,0.03147733211517334
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,7,0.027434666951497395
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,15,0.02741866558790207
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,31,0.027450665831565857
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,63,0.03162666658560435
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,63,0.027664000789324444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,15,0.031770666440327965
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,31,0.03153600047032038
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,127,0.035690667728583016
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,127,0.027989332874615986
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,255,0.052570665876070656
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,255,0.03655466685692469
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,1,0.05505066613356272
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,1,0.04590400060017904
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,3,0.054666668176651
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,3,0.04613333443800608
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,7,0.05499200026194254
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,7,0.04643733302752177
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,15,0.055957332253456116
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,15,0.04626133541266123
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,31,0.05436266462008158
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,31,0.04608533283074697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,63,0.05421333511670431
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,63,0.04608533283074697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,127,0.06004266440868378
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,127,0.050794666012128196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,255,0.09264533718427022
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,255,0.06041066845258077
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,1,0.10175999999046326
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,1,0.09302399555842082
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,3,0.0932426651318868
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,3,0.1030399998029073
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,7,0.10268800457318623
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,7,0.09315199653307597
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,15,0.10211199522018433
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,15,0.09283733367919922
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,31,0.1032319962978363
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,31,0.09311999877293904
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,63,0.10314666231473286
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,63,0.09277333815892537
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,127,0.1092800001303355
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,127,0.09353599945704143
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,255,0.17343467473983765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,1,0.17924267053604126
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,1,0.15473600228627524
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,255,0.11075199643770854
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,3,0.15486933787663779
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,7,0.1785279909769694
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,3,0.17747199535369873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,7,0.15473600228627524
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,15,0.1551093359788259
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,15,0.1768959959348043
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,31,0.17778666814168295
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,63,0.17764800786972046
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,63,0.15570666392644247
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,1,0.009061333412925402
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,3,0.009258666386206945
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,7,0.00926399976015091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,7,0.009098666409651438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,15,0.009136000027259191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,127,0.18875734011332193
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,31,0.15458133816719055
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,15,0.009189333145817121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,1,0.00895999992887179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,31,0.009072000160813332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,63,0.008986666798591614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,3,0.009365333244204521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,127,0.009103999783595404
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,127,0.008943999807039896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,255,0.008949333180983862
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,255,0.010181333248813948
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,511,0.010832000523805618
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,1023,0.009706666693091393
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,1023,0.010949333508809408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,2047,0.012847999731699625
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,2047,0.012928000340859095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,4095,0.013994666437307993
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,63,0.009136000027259191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,4095,0.013056000073750814
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,8191,0.01534933348496755
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,8191,0.0163680004576842
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,16383,0.017269333203633625
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,16383,0.03150933235883713
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,127,0.1570133368174235
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,32767,0.050330668687820435
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,32767,0.033514666060606636
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,65535,0.08747733632723491
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,1,0.009008000294367472
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,65535,0.0524586687485377
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,3,0.00890666681031386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,3,0.009050666665037474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,7,0.008949333180983862
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,7,0.009072000160813332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,15,0.009072000160813332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,15,0.00891733355820179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,31,0.009178666397929192
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,511,0.010981333752473196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,31,0.009114666531483332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,63,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,63,0.00926399976015091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,127,0.008997333546479544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,127,0.009066666786869368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,255,0.008853333070874214
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,255,0.00927466650803884
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,511,0.00961599995692571
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,511,0.010960000256697336
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,1023,0.010879999647537867
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,1023,0.011381333072980246
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,2047,0.013072000195582708
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,2047,0.011760000139474869
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,4095,0.015168000012636185
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,4095,0.013280000537633896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,8191,0.029866665601730347
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,31,0.009408000235756239
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,16383,0.0315786674618721
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,32767,0.0867039958635966
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,32767,0.04990933338801066
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,65535,0.16054399808247885
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,1,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,65535,0.08898133039474487
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,1,0.008922666932145754
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,1,0.00938666673998038
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,3,0.009242666885256767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,3,0.00891733355820179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,7,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,7,0.009162666896979014
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,15,0.008858666444818178
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,15,0.012165332833925882
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,31,0.00898133342464765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,63,0.009621333330869675
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,63,0.011045332998037338
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,127,0.010112000008424124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,255,0.009637333452701569
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,255,0.010960000256697336
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,511,0.01129066695769628
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,16383,0.049285332361857094
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,8191,0.017050666113694508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,511,0.01099733387430509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,1023,0.01293333371480306
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,1023,0.01173866664369901
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,2047,0.01403733342885971
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,2047,0.013066666821638743
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,4095,0.015066667149464289
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,4095,0.03013866643110911
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,8191,0.047872001926104225
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,8191,0.031285333136717476
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,16383,0.08474133412043254
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,16383,0.04974933465321859
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,32767,0.1569866637388865
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,31,0.009349333122372627
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,32767,0.08754666646321614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,1,0.010970667004585266
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,127,0.009253333633144697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,65535,0.15967466433842978
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,3,0.010869332899649939
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,3,0.011071999867757162
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,7,0.009930666536092758
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,7,0.011002667248249054
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,15,0.010970667004585266
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,15,0.011002667248249054
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,255,0.191594660282135
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,31,0.009898666913310686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,31,0.010992000500361124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,255,0.31704533100128174
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,63,0.009872000043590864
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,63,0.010960000256697336
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,127,0.010890666395425797
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,127,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,255,0.01089599976936976
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,511,0.011039999624093374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,1023,0.01312000056107839
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,1023,0.013232000172138214
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,2047,0.0295413335164388
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,4095,0.029989334444204967
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,2047,0.015306666493415833
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,4095,0.04719999929269155
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,1,0.010149333626031876
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,8191,0.08364799618721008
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,8191,0.04880533119042715
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,16383,0.08489066362380981
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,16383,0.1552906632423401
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,1,0.00916800027092298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,32767,0.29946666955947876
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,32767,0.15702399611473083
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,1,0.009130666653315226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,65535,0.5873759984970093
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,65535,0.30101333061854046
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,3,0.009290666629870733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,3,0.009093333035707474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,7,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,15,0.00927466650803884
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,31,0.009119999905427298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,31,0.009173333023985228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,7,0.009269333134094873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,63,0.008933333059151968
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,63,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,127,0.00891733355820179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,127,0.009008000294367472
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,255,0.009173333023985228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,255,0.009114666531483332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,511,0.010847999403874079
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,511,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,1023,0.011178666104873022
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,65535,0.30165332555770874
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,1023,0.011098666737476984
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,2047,0.012970666090647379
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,2047,0.011301333705584208
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,4095,0.01515199989080429
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,8191,0.030218665798505146
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,4095,0.013295999417702356
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,8191,0.017136000096797943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,16383,0.04850666721661886
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,32767,0.08662399649620056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,16383,0.03161599983771642
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,1,0.009098666409651438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,1,0.009130666653315226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,65535,0.16109866897265115
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,65535,0.08927999933560689
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,3,0.010847999403874079
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,15,0.00914666677514712
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,7,0.009375999992092451
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,15,0.010117333382368088
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,7,0.009285333255926767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,15,0.009072000160813332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,31,0.009258666386206945
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,31,0.00938666673998038
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,63,0.009317333499590555
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,127,0.009178666397929192
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,127,0.009253333633144697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,255,0.00949866697192192
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,255,0.009226666763424873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,511,0.01102399950226148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,511,0.01097600037852923
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,1023,0.011365332951148352
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,1023,0.011749333391586939
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,255,0.010778666784365972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,511,0.011098666737476984
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,2047,0.013295999417702356
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,2047,0.013114667187134424
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,4095,0.030154667794704437
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,4095,0.016341333587964375
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,32767,0.050245334704717
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,8191,0.04840533435344696
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,8191,0.031354665756225586
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,16383,0.08669333656628926
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,16383,0.05007466673851013
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,3,0.009183999771873156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,32767,0.08660800258318584
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,1,0.010970667004585266
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,32767,0.15968533356984457
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,1,0.010933333386977514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,3,0.010330666477481524
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,7,0.00983466642598311
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,65535,0.3060320019721985
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,7,0.010965333630641302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,65535,0.16404799620310465
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,15,0.01116266722480456
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,15,0.010415999839703241
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,31,0.010842667271693548
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,31,0.010181333248813948
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,63,0.01109333336353302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,63,0.01081066702802976
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,127,0.010805333654085795
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,127,0.01081066702802976
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,255,0.01102399950226148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,255,0.01099733387430509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,511,0.011525332927703857
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,511,0.010981333752473196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,2047,0.029733332494894665
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,1023,0.013210666676362356
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,2047,0.015178666760524115
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,4095,0.03018666555484136
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,4095,0.048063998421033226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,8191,0.084389328956604
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,8191,0.0487306664387385
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,16383,0.15680000185966492
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,16383,0.08583999673525493
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,1,0.009039999917149544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,32767,0.30106133222579956
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,32767,0.15979733069737753
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,1,0.008976000050703684
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,3,0.010725333044926325
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,63,0.009130666653315226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,7,0.009253333633144697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,65535,0.5880266825358073
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,15,0.009098666409651438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,15,0.00898133342464765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,65535,0.3073439995447795
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,63,0.009029333169261614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,31,0.008933333059151968
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,127,0.009002666920423508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,127,0.009226666763424873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,63,0.009525333220760027
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,255,0.009103999783595404
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,255,0.008874666566650072
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,1023,0.013141332815090815
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,511,0.010773333410422007
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,511,0.010858666151762009
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,1023,0.009269333134094873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,2047,0.010805333654085795
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,2047,0.010762666662534079
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,1023,0.009093333035707474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,4095,0.011002667248249054
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,4095,0.011077333241701126
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,8191,0.015087999403476715
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,8191,0.01310933381319046
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,16383,0.01915733392039935
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,16383,0.019205333044131596
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,32767,0.027349332968393963
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,32767,0.02731200059254964
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,65535,0.031285333136717476
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,65535,0.03170666595300039
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,131071,0.033488000432650246
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,131071,0.03145600110292435
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,1,0.008986666798591614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,1,0.008986666798591614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,3,0.00922133338948091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,3,0.00895999992887179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,7,0.009125333279371262
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,7,0.009194666519761086
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,15,0.009253333633144697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,15,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,31,0.008949333180983862
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,3,0.01007466639081637
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,31,0.00926399976015091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,63,0.00891733355820179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,63,0.00898133342464765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,3,0.009370666618148485
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,127,0.00901333304742972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,127,0.009072000160813332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,255,0.009189333145817121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,511,0.009072000160813332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,1023,0.00922133338948091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,1023,0.009269333134094873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,2047,0.009322666873534521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,2047,0.009136000027259191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,31,0.009354666496316591
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,4095,0.011007999380429586
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,4095,0.011194666226704916
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,8191,0.015141333142916361
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,8191,0.015018666783968607
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,16383,0.019280000279347103
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,16383,0.019354666272799175
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,32767,0.02279466638962428
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,32767,0.02242133269707362
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,65535,0.025450666745503742
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,65535,0.023589332898457844
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,131071,0.04292266567548116
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,131071,0.025674665967623394
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,1,0.009082666908701261
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,1,0.009098666409651438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,3,0.009178666397929192
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,3,0.00891733355820179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,7,0.008965333302815756
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,7,0.00903466654320558
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,15,0.00914666677514712
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,15,0.009162666896979014
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,31,0.009402666861812273
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,31,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,63,0.009279999881982803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,127,0.010005333150426546
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,127,0.00984533317387104
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,255,0.009130666653315226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,255,0.009695999945203463
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,511,0.011087999989589056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,1023,0.011306667079528173
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,1023,0.010885333021481832
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,7,0.00897066667675972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,2047,0.011061333119869232
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,2047,0.011007999380429586
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,255,0.00901333304742972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,511,0.009610666582981745
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,4095,0.01099733387430509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,4095,0.010970667004585266
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,8191,0.015205333630243937
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,8191,0.01524266724785169
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,16383,0.017269333203633625
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,16383,0.01710933322707812
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,32767,0.01977066695690155
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,32767,0.01922133316596349
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,65535,0.03676266719897588
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,65535,0.021136000752449036
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,131071,0.0551093320051829
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,1,0.008863999818762144
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,131071,0.037834666669368744
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,3,0.009088000282645226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,3,0.00892800030608972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,7,0.009194666519761086
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,7,0.009103999783595404
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,15,0.009173333023985228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,31,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,31,0.008901333436369896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,63,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,127,0.009039999917149544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,127,0.00903466654320558
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,255,0.009072000160813332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,255,0.009285333255926767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,511,0.009397333487868309
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,511,0.011050666371981302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,1023,0.009072000160813332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,1023,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,2047,0.010944000134865442
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,2047,0.010960000256697336
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,63,0.009109333157539368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,4095,0.01157333329319954
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,8191,0.015034666905800501
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,8191,0.014943999548753103
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,16383,0.019381333142518997
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,16383,0.019173332800467808
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,32767,0.02312533309062322
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,65535,0.02497600018978119
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,65535,0.023157333334287006
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,131071,0.043840001026789345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,131071,0.025546667476495106
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,1,0.00890666681031386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,1,0.009050666665037474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,3,0.00901333304742972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,3,0.009253333633144697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,7,0.00903466654320558
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,1,0.009098666409651438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,7,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,15,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,15,0.009152000149091085
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,15,0.008933333059151968
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,31,0.009317333499590555
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,63,0.008965333302815756
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,63,0.008858666444818178
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,127,0.00901333304742972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,127,0.009183999771873156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,255,0.009002666920423508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,255,0.009328000247478485
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,511,0.009946666657924652
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,1023,0.009045333291093508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,4095,0.01101333275437355
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,1023,0.010949333508809408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,2047,0.011045332998037338
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,2047,0.011007999380429586
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,4095,0.010965333630641302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,4095,0.011120000233252844
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,32767,0.021354667842388153
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,8191,0.015077333897352219
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,16383,0.017269333203633625
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,16383,0.017194667210181553
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,32767,0.020634666085243225
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,32767,0.019205333044131596
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,65535,0.037402667105197906
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,65535,0.02059200033545494
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,131071,0.05442666510740916
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,131071,0.03763733307520548
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,1,0.009050666665037474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,1,0.00890666681031386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,3,0.009039999917149544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,63,0.009103999783595404
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,3,0.009226666763424873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,7,0.00898133342464765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,7,0.009098666409651438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,15,0.00915733352303505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,15,0.009455999980370203
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,511,0.009861333295702934
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,31,0.009039999917149544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,31,0.009002666920423508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,63,0.009392000113924345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,127,0.009402666861812273
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,127,0.009402666861812273
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,255,0.009125333279371262
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,255,0.00897066667675972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,511,0.010847999403874079
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,511,0.010965333630641302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,1023,0.009296000003814697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,1023,0.009717333440979322
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,8191,0.015189333508412043
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,2047,0.010981333752473196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,2047,0.01109333336353302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,4095,0.013199999928474426
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,4095,0.013072000195582708
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,8191,0.015040000279744467
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,8191,0.015205333630243937
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,16383,0.017429333180189133
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,16383,0.017290666699409485
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,32767,0.01926400015751521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,32767,0.03379733363787333
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,65535,0.05213333169619242
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,65535,0.03533333291610082
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,131071,0.05295466880003611
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,131071,0.08779199918111165
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,1,0.011050666371981302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,1,0.009018666421373686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,31,0.009130666653315226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,3,0.009072000160813332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,3,0.009296000003814697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,7,0.00892800030608972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,7,0.009178666397929192
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,15,0.009279999881982803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,31,0.009205333267649015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,15,0.009173333023985228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,31,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,63,0.009370666618148485
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,63,0.009317333499590555
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,127,0.010885333021481832
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,127,0.009066666786869368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,63,0.009136000027259191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,255,0.009258666386206945
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,255,0.010538666198650995
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,511,0.01099733387430509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,511,0.010933333386977514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,1023,0.012954667210578918
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,1023,0.01119999960064888
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,511,0.011029332876205444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,2047,0.015125333021084467
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,2047,0.013242666920026144
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,4095,0.030229332546393078
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,8191,0.04780266682306925
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,4095,0.015317333241303762
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,8191,0.03139200061559677
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,16383,0.050885334610939026
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,16383,0.08764266967773438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,1,0.00980266680320104
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,3,0.01090666651725769
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,32767,0.16023466984430948
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,7,0.011328000575304031
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,1,0.010970667004585266
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,7,0.01137599969903628
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,3,0.010693332801262537
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,15,0.01108266661564509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,63,0.01098666712641716
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,31,0.009786666681369146
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,63,0.011077333241701126
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,31,0.010351999973257383
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,127,0.011050666371981302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,32767,0.0886293351650238
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,255,0.011039999624093374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,255,0.011125333607196808
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,511,0.01116266722480456
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,511,0.011018666128317514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,1023,0.013157332936922709
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,1023,0.013034666577974955
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,4095,0.04781333108743032
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,2047,0.029450667401154835
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,2047,0.015098666151364645
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,4095,0.03084266682465871
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,8191,0.04948266843954722
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,8191,0.08489066362380981
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,1,0.011370666325092316
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,15,0.009984000275532404
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,16383,0.15971199671427408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,16383,0.08617066343625386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,32767,0.3059626619021098
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,1,0.011231999844312668
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,32767,0.1604746679464976
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,127,0.011029332876205444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,3,0.011317333827416102
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,3,0.011120000233252844
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,7,0.011071999867757162
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,15,0.011989332735538483
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,31,0.010954666882753372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,7,0.011109333485364914
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,15,0.01089599976936976
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,63,0.011061333119869232
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,127,0.01126933346192042
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,255,0.01099733387430509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,511,0.014933332800865173
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,511,0.013151999562978745
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,1023,0.029461334149042766
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,1023,0.015157333264748255
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,2047,0.04832000037034353
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,2047,0.03143999973932902
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,4095,0.08467732866605122
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,4095,0.04971733192602793
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,8191,0.15850133697191873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,31,0.01118933285276095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,63,0.01108266661564509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,16383,0.30188266436258954
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,1,0.008869333192706108
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,127,0.011328000575304031
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,16383,0.16390400131543478
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,255,0.011168000598748526
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,3,0.00890666681031386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,32767,0.5922559897104899
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,7,0.010191999996701876
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,15,0.009392000113924345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,7,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,31,0.00926399976015091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,31,0.008869333192706108
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,63,0.009088000282645226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,15,0.009423999736706415
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,63,0.009253333633144697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,127,0.00916800027092298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,127,0.009194666519761086
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,255,0.009050666665037474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,255,0.009226666763424873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,32767,0.3105280001958211
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,511,0.010911999891201654
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,511,0.010944000134865442
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,1023,0.009317333499590555
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,1023,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,2047,0.010890666395425797
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,4095,0.011055999745925268
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,4095,0.011087999989589056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,8191,0.015333333363135656
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,8191,0.015200000256299973
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,2047,0.009045333291093508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,32767,0.019573333362738293
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,8191,0.08799999952316284
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,32767,0.019082666685183842
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,65535,0.03728533287843069
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,65535,0.02083733429511388
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,131071,0.03948266555865606
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,1,0.008954666554927826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,3,0.009002666920423508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,3,0.008965333302815756
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,7,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,7,0.009253333633144697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,15,0.008992000172535578
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,15,0.00892800030608972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,1,0.009018666421373686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,31,0.009216000015536943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,31,0.00960533320903778
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,63,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,63,0.011173332730929056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,127,0.00895999992887179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,127,0.009381333366036415
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,255,0.01002133327225844
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,255,0.009189333145817121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,1023,0.009253333633144697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,511,0.011322667201360067
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,1023,0.009258666386206945
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,2047,0.01108266661564509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,2047,0.01098666712641716
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,4095,0.01331199953953425
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,4095,0.01302933320403099
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,16383,0.017349333812793095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,8191,0.01522133375207583
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,8191,0.015077333897352219
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,16383,0.017210666090250015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,131071,0.054789334535598755
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,16383,0.01710933322707812
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,1,0.008816000074148178
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,32767,0.03362133353948593
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,32767,0.019333332777023315
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,65535,0.05231466889381409
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,131071,0.08932800094286601
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,131071,0.05442666510740916
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,1,0.00890666681031386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,1,0.009477333476146063
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,3,0.009205333267649015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,3,0.008965333302815756
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,3,0.009119999905427298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,7,0.009429333110650381
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,15,0.009082666908701261
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,15,0.009109333157539368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,31,0.009141333401203156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,31,0.009285333255926767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,63,0.009136000027259191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,63,0.009354666496316591
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,127,0.008997333546479544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,511,0.009663999701539675
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,255,0.00915733352303505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,255,0.009583999713261923
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,511,0.011146667102972666
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,1023,0.011061333119869232
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,511,0.01119999960064888
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,1023,0.010874666273593903
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,16383,0.018965333700180054
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,2047,0.012970666090647379
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,2047,0.013167999684810638
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,4095,0.01532799998919169
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,4095,0.013786666095256805
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,8191,0.0164533331990242
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,8191,0.015583999454975128
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,16383,0.03156266609827677
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,16383,0.01747200017174085
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,32767,0.04994666576385498
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,65535,0.03543466577927271
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,32767,0.03388266762097677
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,65535,0.0865226686000824
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,65535,0.052426666021347046
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,131071,0.15810666481653848
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,131071,0.08975467085838318
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,7,0.009082666908701261
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,1,0.011077333241701126
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,3,0.010821333775917688
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,1,0.011354666203260422
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,7,0.010863999525705973
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,3,0.010938666760921478
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,127,0.009594666461149851
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,7,0.01090666651725769
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,15,0.010858666151762009
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,15,0.011029332876205444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,31,0.010970667004585266
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,63,0.010960000256697336
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,63,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,31,0.011391999820868174
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,127,0.013839999834696451
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,127,0.011125333607196808
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,255,0.011050666371981302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,255,0.010890666395425797
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,511,0.011114666859308878
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,511,0.011215999722480774
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,1023,0.014896000425020853
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,2047,0.015263999501864115
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,2047,0.029520000020662945
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,8191,0.049866666396458946
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,1023,0.013503999759753546
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,4095,0.04901866614818573
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,4095,0.03173866619666418
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,1,0.011370666325092316
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,1,0.011754666765530905
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,3,0.011424000064531961
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,7,0.01119999960064888
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,3,0.011498666057984034
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,15,0.011343999455372492
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,7,0.011349332829316458
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,8191,0.08690667152404785
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,63,0.011802667131026586
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,63,0.011328000575304031
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,31,0.01137599969903628
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,31,0.01198400060335795
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,16383,0.08726400136947632
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,127,0.011701333026091257
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,255,0.01145600030819575
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,127,0.011365332951148352
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,511,0.01505600040157636
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,15,0.011610666910807291
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,16383,0.1618880033493042
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,511,0.013264000415802002
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,1023,0.029765332738558452
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,1023,0.01611199975013733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,4095,0.08621866504351298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,2047,0.03133866687615713
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,4095,0.0506933331489563
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,2047,0.04971733192602793
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,8191,0.16051200032234192
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,8191,0.08742400010426839
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,1,0.01341333364446958
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,1,0.014005333185195923
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,3,0.013290667285521826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,7,0.01504533365368843
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,16383,0.307584007581075
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,7,0.013253333667914072
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,3,0.015168000012636185
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,15,0.01443733274936676
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,15,0.013306666165590286
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,31,0.013301332791646322
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,63,0.01516266663869222
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,31,0.015216000378131866
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,63,0.013183999806642532
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,127,0.015098666151364645
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,127,0.01310933381319046
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,255,0.015146666516860327
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,255,0.013093333691358566
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,255,0.011109333485364914
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,511,0.030165334542592365
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,511,0.017152000218629837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,16383,0.16357333461443582
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,1023,0.04886400202910105
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,1023,0.0317546675602595
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,2047,0.08525866270065308
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,2047,0.05034666756788889
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,8191,0.3081493377685547
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,1,0.011551999797423681
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,8191,0.16104533274968466
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,1,0.010992000500361124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,16383,0.3081493377685547
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,3,0.01108266661564509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,3,0.011226666470368704
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,7,0.011418666690587997
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,16383,0.5979306697845459
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,7,0.011247999966144562
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,15,0.011424000064531961
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,4095,0.15878400206565857
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,15,0.011194666226704916
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,31,0.01138666644692421
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,4095,0.08750933408737183
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,31,0.01109333336353302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,63,0.011253333340088526
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,127,0.011546666423479715
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,127,0.010992000500361124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,255,0.01109333336353302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,63,0.011247999966144562
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,511,0.01470400020480156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,511,0.01320533330241839
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,1023,0.03075733284155528
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,1023,0.015184000134468079
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,1,0.015157333264748255
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,2047,0.04830400149027506
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,1,0.013381333400805792
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,2047,0.03148799886306127
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,3,0.014858666807413101
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,4095,0.05013333261013031
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,4095,0.08635733524958293
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,255,0.01146666705608368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,7,0.013306666165590286
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,15,0.013295999417702356
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,7,0.014991999914248785
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,15,0.014912000546852747
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,63,0.015200000256299973
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,63,0.013248000293970108
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,127,0.014965333044528961
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,127,0.014032000054915747
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,255,0.014858666807413101
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,31,0.01509333277742068
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,31,0.01312000056107839
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,3,0.01368533323208491
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,255,0.013359999905029932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,511,0.030389333764712017
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,511,0.017258666455745697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,4095,0.1604639987150828
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,1023,0.04924266537030538
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,1023,0.031632001201311745
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,4095,0.08803199728329976
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,1,0.019167999426523846
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,1,0.017594666530688603
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,3,0.019120000302791595
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,2047,0.08743466933568318
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,2047,0.04995200037956238
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,3,0.01720533271630605
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,15,0.019050666441520054
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,7,0.01739199956258138
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,7,0.019280000279347103
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,31,0.01945066700379054
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,63,0.01711999997496605
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,127,0.019194666296243668
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,127,0.01757866640885671
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,255,0.01728533332546552
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,255,0.030085332691669464
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,31,0.017152000218629837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,15,0.017349333812793095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,63,0.01933866615096728
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,511,0.04849599798520406
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,511,0.03133333226044973
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,2047,0.15656532843907675
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,2047,0.0890826682249705
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,1023,0.05090666810671488
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,1023,0.08536000053087871
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,4095,0.16225600242614746
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,4095,0.29982932408650714
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,1,0.015029333531856537
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,3,0.015210667004187902
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,15,0.013290667285521826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,7,0.013477332890033722
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,15,0.015008000036080679
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,3,0.01333333303531011
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,31,0.014917333920796713
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,7,0.01534933348496755
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,31,0.01331199953953425
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,1,0.013365333278973898
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,63,0.015018666783968607
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,63,0.014005333185195923
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,127,0.015237333873907724
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,127,0.013653332988421122
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,255,0.01526933287580808
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,255,0.013733333597580591
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,511,0.03125333289305369
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,511,0.017114666601022083
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,1023,0.04990399877230326
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,1023,0.031925333042939506
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,1,0.019530666371186573
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,1,0.017152000218629837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,2047,0.0867733359336853
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,2047,0.05152533451716105
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,3,0.019285333653291065
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,3,0.0174346665541331
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,7,0.019093333433071773
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,7,0.01720000058412552
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,15,0.019386666516462963
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,15,0.017173333714405697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,63,0.019280000279347103
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,63,0.017322666943073273
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,127,0.019194666296243668
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,127,0.01730666682124138
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,31,0.019215999792019527
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,255,0.017184000462293625
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,255,0.029829333225886028
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,31,0.017263999829689663
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,511,0.0491893341143926
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,511,0.03199466566244761
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,1,0.02756800005833308
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,1,0.030810666580994923
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,1023,0.0851093331972758
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,1023,0.050757333636283875
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,3,0.03146666785081228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,3,0.027834666272004444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,7,0.029978667696317036
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,7,0.027488000690937042
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,2047,0.0883733332157135
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,2047,0.1601599951585134
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,31,0.031386665999889374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,15,0.03162666658560435
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,31,0.02735466758410136
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,63,0.0315733328461647
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,63,0.02755733331044515
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,15,0.027653334041436512
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,255,0.051781331499417625
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,255,0.03571200122435888
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,511,0.08675199747085571
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,511,0.05349866549173991
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,127,0.027450665831565857
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,127,0.035631999373435974
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,2047,0.16507200400034586
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,2047,0.3027946750322978
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,1023,0.1595253348350525
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,1,0.008965333302815756
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,15,0.009205333267649015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,7,0.009008000294367472
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,1023,0.09105066458384196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,3,0.009258666386206945
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,31,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,7,0.00903466654320558
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,3,0.008976000050703684
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,31,0.009808000177145004
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,63,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,127,0.009173333023985228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,255,0.00922133338948091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,1,0.010138666878143946
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,255,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,511,0.00916800027092298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,511,0.011018666128317514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,1023,0.009285333255926767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,1023,0.011087999989589056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,2047,0.01097600037852923
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,2047,0.01097600037852923
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,4095,0.013072000195582708
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,8191,0.015290666371583939
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,8191,0.015285332997639975
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,16383,0.017157333592573803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,16383,0.016010666886965435
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,32767,0.03429866582155228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,32767,0.019253333409627277
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,65535,0.05193600058555603
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,65535,0.03457599878311157
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,131071,0.054058666030565895
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,131071,0.09034132957458496
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,1,0.00914666677514712
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,3,0.008997333546479544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,3,0.008938666433095932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,7,0.009328000247478485
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,7,0.009136000027259191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,15,0.008949333180983862
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,15,0.00879466657837232
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,31,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,31,0.00895999992887179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,15,0.009152000149091085
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,63,0.00892800030608972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,63,0.008997333546479544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,127,0.009072000160813332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,127,0.009050666665037474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,127,0.009279999881982803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,63,0.00901333304742972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,255,0.009242666885256767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,255,0.009408000235756239
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,511,0.00914666677514712
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,511,0.01098666712641716
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,1023,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,2047,0.013162666310866674
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,2047,0.014053333550691605
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,4095,0.013221333424250284
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,8191,0.01709866647919019
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,8191,0.015210667004187902
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,16383,0.032399999598662056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,16383,0.017557332913080852
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,32767,0.0334346666932106
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,32767,0.050714666644732155
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,65535,0.08694400389989217
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,1,0.008965333302815756
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,65535,0.05183466772238413
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,131071,0.16101866960525513
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,1,0.009861333295702934
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,1,0.009045333291093508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,131071,0.090938667456309
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,3,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,3,0.009125333279371262
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,7,0.00933333362142245
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,7,0.009130666653315226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,15,0.009066666786869368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,15,0.00901333304742972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,31,0.009205333267649015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,31,0.009103999783595404
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,63,0.009839999799927076
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,63,0.009183999771873156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,127,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,127,0.009109333157539368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,255,0.009450666606426239
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,255,0.00983466642598311
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,511,0.011130666981140772
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,511,0.011066666493813196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,1023,0.01098666712641716
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,1023,0.010911999891201654
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,1023,0.011146667102972666
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,2047,0.013061333447694778
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,2047,0.012074666718641916
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,4095,0.01368533323208491
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,4095,0.015135999768972397
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,4095,0.012960000584522883
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,8191,0.029706666866938274
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,8191,0.016410666207472484
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,16383,0.04842133323351542
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,32767,0.08369599779446919
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,32767,0.04987200101216634
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,65535,0.08821333448092143
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,65535,0.15763200322786966
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,131071,0.30320000648498535
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,131071,0.16263467073440552
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,4095,0.012954667210578918
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,16383,0.03155199935038885
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,1,0.019285333653291065
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,1,0.01714666684468587
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,3,0.0174346665541331
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,3,0.019343999524911244
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,7,0.019541333119074505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,7,0.01729600007335345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,15,0.0191040001809597
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,15,0.017525333911180496
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,31,0.019461333751678467
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,63,0.01924266666173935
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,63,0.017290666699409485
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,127,0.019285333653291065
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,127,0.01716800034046173
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,31,0.017407999684413273
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,255,0.03046400099992752
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,255,0.017375999440749485
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,511,0.04960533479849497
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,511,0.03145066648721695
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,1,0.03150933235883713
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,1023,0.050240000089009605
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,1023,0.08709333340326945
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,1,0.027637332677841187
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,3,0.0316746657093366
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,3,0.027637332677841187
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,7,0.031583999594052635
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,15,0.03152533372243246
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,7,0.0273333340883255
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,31,0.028330666323502857
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,15,0.027434666951497395
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,63,0.03147733211517334
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,31,0.031680000325044
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,63,0.027600000301996868
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,127,0.03545066714286804
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,127,0.02736533433198929
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,255,0.0516533354918162
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,255,0.03563733398914337
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,511,0.08876799543698628
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,1,0.05721066892147064
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,1023,0.16056000192960104
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,1023,0.09051733215649922
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,1,0.04785066843032837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,3,0.05634133517742157
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,3,0.047930667797724404
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,7,0.05659733215967814
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,511,0.05388799806435903
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,7,0.04795733094215393
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,31,0.057722667853037514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,31,0.04808000226815542
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,63,0.05745600163936615
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,63,0.04822400212287903
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,127,0.05272000034650167
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,15,0.057146668434143066
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,15,0.04799466828505198
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,127,0.06062933305899302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,255,0.09225066502888997
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,255,0.061386664708455406
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,511,0.16352533300717673
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,511,0.0959999958674113
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,1,0.03136000037193298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,1,0.0276053324341774
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,1023,0.30773866176605225
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,1023,0.16880534092585245
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,3,0.03160000095764796
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,3,0.028069332242012024
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,7,0.031541332602500916
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,7,0.027653334041436512
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,15,0.031370667119820915
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,15,0.027488000690937042
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,63,0.0276053324341774
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,63,0.03143999973932902
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,127,0.03579733272393545
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,127,0.027978666126728058
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,31,0.027600000301996868
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,31,0.031445334355036415
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,255,0.05186666548252106
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,255,0.03586666782697042
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,511,0.05385066568851471
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,1,0.05541866521040598
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,1,0.0460746685663859
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,511,0.08979733784993489
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,3,0.04590400060017904
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,3,0.055434669057528176
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,7,0.05495999753475189
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,7,0.04604266583919525
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,15,0.05434666574001312
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,15,0.04603200157483419
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,31,0.05455466608206431
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,31,0.04671466847260793
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,63,0.04586133360862732
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,63,0.05514666438102722
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,127,0.06014933188756307
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,127,0.050330668687820435
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,255,0.09076266487439473
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,255,0.06046399970849355
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,1,0.09475732843081157
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,1,0.10337600111961365
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,511,0.16450666387875876
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,511,0.09387200077374776
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,3,0.10173867146174113
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,3,0.09243200222651164
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,7,0.10142399867375691
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,7,0.09296000003814697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,15,0.10179199775060017
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,15,0.09302399555842082
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,63,0.1032533347606659
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,63,0.09300800164540608
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,31,0.10297600428263347
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,31,0.09288000067075093
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,1,0.00897066667675972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,3,0.009279999881982803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,1,0.009253333633144697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,3,0.00996800015370051
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,7,0.00891733355820179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,7,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,15,0.008965333302815756
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,15,0.009306666751702627
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,127,0.10957866907119751
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,31,0.008826666822036108
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,31,0.009882666791478792
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,63,0.009109333157539368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,127,0.009189333145817121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,127,0.008901333436369896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,127,0.09459199508031209
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,255,0.00903466654320558
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,255,0.009269333134094873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,511,0.010768000036478043
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,511,0.010965333630641302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,1023,0.010954666882753372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,2047,0.012965332716703415
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,2047,0.013050666699806849
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,4095,0.015040000279744467
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,4095,0.01301866645614306
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,63,0.008869333192706108
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,8191,0.017194667210181553
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,8191,0.015018666783968607
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,16383,0.03276266654332479
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,1023,0.011146667102972666
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,16383,0.017184000462293625
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,32767,0.05081599950790405
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,32767,0.03352533280849457
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,65535,0.08898666501045227
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,65535,0.05253333350022634
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,131071,0.16354667147000632
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,131071,0.0897813340028127
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,1,0.009141333401203156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,1,0.00922133338948091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,3,0.009002666920423508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,3,0.009173333023985228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,7,0.009018666421373686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,7,0.009322666873534521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,15,0.009162666896979014
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,31,0.009050666665037474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,31,0.009056000038981438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,63,0.00891733355820179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,255,0.17217065890630087
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,63,0.00898133342464765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,255,0.11176000038782756
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,127,0.009194666519761086
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,127,0.009056000038981438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,255,0.009152000149091085
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,511,0.009365333244204521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,1023,0.011066666493813196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,1023,0.010965333630641302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,2047,0.013056000073750814
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,2047,0.011039999624093374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,4095,0.015087999403476715
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,4095,0.012986666212479273
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,8191,0.015360000232855478
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,8191,0.02945599953333537
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,16383,0.04849599798520406
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,16383,0.03165333221356074
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,32767,0.08514133095741272
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,32767,0.04948266843954722
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,15,0.009253333633144697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,65535,0.1597760021686554
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,65535,0.0872373382250468
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,1,0.009125333279371262
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,1,0.00915733352303505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,131071,0.3054773410161336
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,3,0.010469333579142889
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,3,0.009178666397929192
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,255,0.009152000149091085
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,131071,0.16591466466585794
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,511,0.3154346744219462
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,7,0.008997333546479544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,511,0.011087999989589056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,15,0.009050666665037474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,15,0.009008000294367472
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,31,0.008912000184257826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,31,0.010122666756312052
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,63,0.009056000038981438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,127,0.009541333342591921
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,255,0.009216000015536943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,255,0.009546666716535887
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,511,0.01126933346192042
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,511,0.011125333607196808
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,1023,0.013130666067202887
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,1023,0.011231999844312668
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,2047,0.013264000415802002
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,2047,0.013242666920026144
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,4095,0.015130666395028433
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,8191,0.0317493329445521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,8191,0.047968000173568726
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,16383,0.08433066805203755
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,16383,0.050373335679372154
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,32767,0.15642666816711426
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,32767,0.08675733208656311
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,7,0.010928000013033548
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,65535,0.3009226719538371
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,65535,0.16135467092196146
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,63,0.009306666751702627
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,1,0.009216000015536943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,1,0.009338666374484697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,131071,0.588149348894755
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,3,0.00914666677514712
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,3,0.00926399976015091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,7,0.008992000172535578
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,15,0.009162666896979014
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,7,0.008949333180983862
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,4095,0.029706666866938274
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,127,0.011077333241701126
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,31,0.009343999748428663
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,63,0.009050666665037474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,127,0.009898666913310686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,63,0.00915733352303505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,127,0.009205333267649015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,255,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,255,0.008943999807039896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,511,0.01109333336353302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,511,0.010778666784365972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,1023,0.010922666639089584
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,1023,0.010949333508809408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,2047,0.012138667205969492
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,2047,0.012970666090647379
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,4095,0.015013333410024643
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,4095,0.01312000056107839
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,8191,0.030159999926884968
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,8191,0.017050666113694508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,16383,0.031541332602500916
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,16383,0.04884799818197886
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,32767,0.0855519970258077
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,131071,0.30855466922124225
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,32767,0.051407997806866966
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,15,0.00895999992887179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,31,0.009242666885256767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,65535,0.1613653302192688
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,65535,0.09211200475692749
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,511,0.17964800198872885
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,1,0.009509333098928133
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,1,0.009088000282645226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,3,0.009973333527644476
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,7,0.009290666629870733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,15,0.009338666374484697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,3,0.009317333499590555
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,131071,0.16788800557454428
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,31,0.009152000149091085
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,31,0.009183999771873156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,63,0.008943999807039896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,127,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,63,0.009434666484594345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,127,0.009317333499590555
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,255,0.00915733352303505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,255,0.00915733352303505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,511,0.011152000476916632
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,511,0.011087999989589056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,1023,0.011349332829316458
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,1023,0.011173332730929056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,2047,0.015125333021084467
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,2047,0.013077333569526672
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,4095,0.030106666187445324
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,4095,0.01543466622630755
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,8191,0.04901333153247833
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,16383,0.0865760048230489
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,8191,0.031685332457224526
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,7,0.009717333440979322
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,131071,0.3069973389307658
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,32767,0.087226668993632
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,1,0.008997333546479544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,32767,0.15945600469907126
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,1,0.009194666519761086
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,65535,0.16079466541608176
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,3,0.009354666496316591
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,131071,0.30956266323725384
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,7,0.009125333279371262
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,15,0.008943999807039896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,3,0.008954666554927826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,131071,0.5990613301595052
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,7,0.009002666920423508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,15,0.009322666873534521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,31,0.008912000184257826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,31,0.00898133342464765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,63,0.009018666421373686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,127,0.00915733352303505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,127,0.00898133342464765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,255,0.009290666629870733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,255,0.009226666763424873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,511,0.011098666737476984
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,511,0.009056000038981438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,1023,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,1023,0.009141333401203156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,2047,0.009050666665037474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,16383,0.04987200101216634
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,2047,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,4095,0.01108266661564509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,4095,0.011066666493813196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,8191,0.013690666606028875
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,8191,0.015141333142916361
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,15,0.009152000149091085
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,16383,0.01887999971707662
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,16383,0.019248000035683315
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,32767,0.027280000348885853
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,32767,0.027285332481066387
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,65535,0.03160000095764796
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,65535,0.030213333666324615
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,131071,0.03341866781314214
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,131071,0.03288000077009201
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,1,0.008965333302815756
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,1,0.009173333023985228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,3,0.009125333279371262
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,7,0.008896000062425932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,7,0.00916800027092298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,15,0.008954666554927826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,31,0.00898133342464765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,31,0.008896000062425932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,63,0.009205333267649015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,63,0.008879999940594038
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,127,0.009114666531483332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,127,0.009194666519761086
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,65535,0.3058346708615621
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,255,0.00897066667675972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,255,0.009002666920423508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,511,0.010805333654085795
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,1023,0.009322666873534521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,1023,0.009082666908701261
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,2047,0.009152000149091085
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,511,0.009226666763424873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,2047,0.011029332876205444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,4095,0.011120000233252844
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,4095,0.011146667102972666
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,8191,0.015114666273196539
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,8191,0.015109332899252573
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,16383,0.019237333287795384
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,16383,0.019253333409627277
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,32767,0.02184533327817917
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,32767,0.02258666604757309
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,65535,0.02367466688156128
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,65535,0.023658665517965954
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,131071,0.04237333436806997
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,131071,0.02536533276240031
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,1,0.008943999807039896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,1,0.00914666677514712
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,3,0.008997333546479544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,7,0.00901333304742972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,7,0.008976000050703684
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,15,0.009039999917149544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,15,0.00903466654320558
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,63,0.009322666873534521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,31,0.009279999881982803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,31,0.008954666554927826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,63,0.008879999940594038
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,63,0.00898133342464765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,127,0.008965333302815756
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,127,0.00916800027092298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,255,0.00903466654320558
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,255,0.009119999905427298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,511,0.010981333752473196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,511,0.009141333401203156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,1023,0.009226666763424873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,1023,0.009050666665037474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,2047,0.01080000028014183
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,3,0.009253333633144697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,2047,0.009253333633144697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,4095,0.011114666859308878
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,4095,0.01108266661564509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,8191,0.015429332852363586
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,15,0.009306666751702627
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,8191,0.014975999792416891
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,16383,0.01926933353145917
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,16383,0.019248000035683315
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,32767,0.02257599929968516
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,65535,0.025194667279720306
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,65535,0.023344000180562336
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,131071,0.04223466912905375
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,1,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,1,0.00922133338948091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,3,0.00901333304742972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,3,0.009178666397929192
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,7,0.009072000160813332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,7,0.00983466642598311
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,15,0.009130666653315226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,15,0.00898133342464765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,31,0.009114666531483332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,31,0.009045333291093508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,63,0.00916800027092298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,63,0.008954666554927826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,127,0.008954666554927826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,127,0.008938666433095932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,255,0.009125333279371262
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,255,0.008949333180983862
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,511,0.011055999745925268
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,511,0.01089599976936976
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,1023,0.009829333052039146
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,1023,0.00926399976015091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,2047,0.010890666395425797
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,2047,0.01091733326514562
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,4095,0.011173332730929056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,4095,0.011018666128317514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,3,0.009242666885256767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,8191,0.015024000157912573
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,16383,0.017312000195185345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,32767,0.020309332758188248
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,32767,0.019173332800467808
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,65535,0.03618666778008143
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,65535,0.02120000123977661
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,131071,0.055013333757718406
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,131071,0.03851733356714249
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,1,0.009109333157539368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,1,0.009189333145817121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,3,0.010293333480755487
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,3,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,7,0.009306666751702627
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,32767,0.02164799968401591
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,7,0.00891733355820179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,15,0.010879999647537867
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,15,0.011045332998037338
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,131071,0.025013332565625507
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,31,0.009098666409651438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,31,0.009103999783595404
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,63,0.008997333546479544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,63,0.011061333119869232
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,127,0.01098666712641716
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,127,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,255,0.009173333023985228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,255,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,511,0.011141333729028702
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,511,0.01089599976936976
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,1023,0.011322667201360067
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,1023,0.011247999966144562
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,8191,0.015125333021084467
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,2047,0.01314666618903478
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,2047,0.013386666774749756
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,16383,0.017184000462293625
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,4095,0.015200000256299973
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,8191,0.04878933231035868
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,8191,0.0314026673634847
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,16383,0.050714666644732155
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,16383,0.08643733461697896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,32767,0.16077333688735962
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,32767,0.08885332942008972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,1,0.011045332998037338
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,1,0.009989333028594652
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,3,0.010687999427318573
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,3,0.009658666948477427
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,65535,0.30762133995691937
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,7,0.01116266722480456
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,7,0.0102613332370917
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,15,0.011055999745925268
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,15,0.011077333241701126
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,65535,0.16197333733240762
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,31,0.01102399950226148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,31,0.011125333607196808
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,63,0.010928000013033548
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,63,0.011136000355084738
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,127,0.01110400011142095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,4095,0.031034665803114574
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,255,0.010266666611035665
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,255,0.011242666592200598
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,511,0.01110400011142095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,127,0.011061333119869232
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,1023,0.013157332936922709
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,4095,0.04823466638724009
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,2047,0.015082667271296183
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,4095,0.03048533449570338
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,8191,0.08493866523106892
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,8191,0.0495306650797526
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,16383,0.1591146687666575
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,16383,0.08647466699282329
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,32767,0.30555200576782227
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,32767,0.15958933035532633
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,1,0.009397333487868309
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,3,0.008885333314538002
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,7,0.008896000062425932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,1,0.009338666374484697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,3,0.009130666653315226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,65535,0.5972799857457479
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,511,0.010970667004585266
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,7,0.009018666421373686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,65535,0.30613332986831665
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,15,0.008976000050703684
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,2047,0.029653333127498627
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,63,0.008938666433095932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,127,0.009152000149091085
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,31,0.00922133338948091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,255,0.008885333314538002
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,255,0.009338666374484697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,511,0.009253333633144697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,511,0.00926399976015091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,1023,0.009642666826645533
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,1023,0.010575999816258749
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,127,0.009194666519761086
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,63,0.009082666908701261
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,2047,0.011114666859308878
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,2047,0.010842667271693548
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,31,0.008912000184257826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,4095,0.01089599976936976
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,8191,0.01534933348496755
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,8191,0.015184000134468079
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,16383,0.017231999586025875
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,16383,0.017114666601022083
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,32767,0.02080533280968666
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,32767,0.01809599995613098
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,65535,0.03749333322048187
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,65535,0.021274665991465252
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,131071,0.054933334390322365
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,1,0.009066666786869368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,131071,0.037903999288876854
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,1,0.009183999771873156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,3,0.008976000050703684
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,3,0.00890666681031386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,7,0.009077333534757296
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,7,0.009322666873534521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,15,0.008965333302815756
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,15,0.009098666409651438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,31,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,31,0.009103999783595404
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,63,0.008965333302815756
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,63,0.009109333157539368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,127,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,127,0.009018666421373686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,255,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,511,0.009162666896979014
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,511,0.009925333162148794
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,1023,0.009482666850090027
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,1023,0.00892800030608972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,2047,0.010981333752473196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,2047,0.010890666395425797
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,15,0.009114666531483332
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,4095,0.013034666577974955
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,8191,0.01516266663869222
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,1023,0.013232000172138214
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,8191,0.015706667055686314
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,16383,0.017130666722853977
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,16383,0.01740266631046931
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,32767,0.019296000401178997
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,32767,0.03357866654793421
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,65535,0.05193600058555603
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,65535,0.03583999971548716
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,131071,0.08959466218948364
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,131071,0.053583999474843345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,4095,0.01089599976936976
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,1,0.011061333119869232
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,1,0.01121066634853681
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,3,0.011066666493813196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,3,0.011120000233252844
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,7,0.01099733387430509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,7,0.010954666882753372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,15,0.011194666226704916
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,15,0.010778666784365972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,31,0.01089599976936976
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,255,0.009205333267649015
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,31,0.010911999891201654
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,63,0.01099733387430509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,63,0.010885333021481832
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,4095,0.013093333691358566
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,127,0.011120000233252844
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,127,0.011002667248249054
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,255,0.01102399950226148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,255,0.010911999891201654
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,511,0.011087999989589056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,511,0.01119999960064888
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,1023,0.013056000073750814
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,1023,0.012800000607967377
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,2047,0.029994666576385498
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,2047,0.014991999914248785
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,4095,0.04877866804599762
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,8191,0.0865226686000824
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,4095,0.03140799949566523
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,8191,0.051594664653142296
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,16383,0.08747733632723491
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,16383,0.15984533230463663
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,1,0.011098666737476984
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,3,0.011018666128317514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,1,0.011301333705584208
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,7,0.011194666226704916
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,15,0.011322667201360067
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,7,0.010890666395425797
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,32767,0.1611733337243398
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,3,0.011306667079528173
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,15,0.01137599969903628
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,32767,0.3086666663487752
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,31,0.011301333705584208
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,31,0.011050666371981302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,63,0.011221333096424738
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,63,0.011258666714032492
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,127,0.011109333485364914
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,255,0.011306667079528173
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,511,0.015034666905800501
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,511,0.013248000293970108
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,127,0.011434666812419891
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,1023,0.029311999678611755
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,1023,0.015040000279744467
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,255,0.010992000500361124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,4095,0.08711466193199158
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,4095,0.05007466673851013
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,2047,0.0484799991051356
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,2047,0.03183999905983607
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,8191,0.1604693333307902
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,8191,0.08841066559155782
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,16383,0.1634986698627472
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,3,0.01166933278242747
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,16383,0.3070133328437805
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,3,0.011237333218256632
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,7,0.011685332904259363
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,1,0.011146667102972666
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,32767,0.6041066646575928
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,7,0.011168000598748526
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,32767,0.314794659614563
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,15,0.012282667060693106
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,15,0.011226666470368704
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,31,0.011071999867757162
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,31,0.011152000476916632
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,63,0.011087999989589056
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,127,0.011317333827416102
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,63,0.012037333101034164
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,255,0.012831999609867731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,255,0.011045332998037338
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,127,0.01145600030819575
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,1,0.011594666788975397
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,511,0.013023999830087027
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,511,0.013610667238632837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,1023,0.03012266755104065
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,1023,0.016741332908471424
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,2047,0.031717332700888314
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,8191,0.16086933016777039
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,2047,0.049728001157442726
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,1,0.015168000012636185
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,8191,0.08875200152397156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,4095,0.050053333242734276
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,4095,0.08672533432642619
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,3,0.01505600040157636
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,1,0.013141332815090815
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,3,0.01328533391157786
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,7,0.015029333531856537
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,7,0.013093333691358566
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,31,0.01515199989080429
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,31,0.013066666821638743
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,63,0.01532799998919169
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,63,0.013797332843144735
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,127,0.015072000523408255
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,127,0.01303999995191892
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,15,0.014885333677132925
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,255,0.015178666760524115
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,255,0.013381333400805792
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,15,0.014335999886194864
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,511,0.01735466718673706
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,511,0.02995733420054118
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,1023,0.04905066887537638
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,1023,0.03165333221356074
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,4095,0.16048533717791238
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,2047,0.04996266464392344
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,2047,0.08635733524958293
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,8191,0.1623253325621287
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,8191,0.313098669052124
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,1,0.014938666174809137
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,1,0.013338666409254074
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,3,0.01509333277742068
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,4095,0.08947199583053589
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,7,0.013221333424250284
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,15,0.013253333667914072
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,31,0.015178666760524115
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,15,0.015130666395028433
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,31,0.01392000044385592
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,7,0.015103999525308609
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,3,0.013167999684810638
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,63,0.015184000134468079
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,127,0.015135999768972397
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,63,0.013471999516089758
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,127,0.013301332791646322
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,255,0.015216000378131866
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,255,0.013370666652917862
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,511,0.030602666238943737
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,511,0.0173333336909612
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,1023,0.05003199974695841
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,1023,0.03173333406448364
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,1,0.019248000035683315
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,1,0.017221332838137943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,2047,0.05026133358478546
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,2047,0.08702400326728821
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,3,0.019274666905403137
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,3,0.01747200017174085
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,7,0.019088000059127808
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,4095,0.09018666545550029
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,7,0.017488000293572743
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,4095,0.16356266538302103
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,31,0.01923199991385142
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,31,0.01749333366751671
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,63,0.019296000401178997
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,15,0.019194666296243668
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,15,0.017322666943073273
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,127,0.019274666905403137
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,255,0.030432000756263733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,255,0.01722666621208191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,511,0.0491839994986852
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,511,0.032474666833877563
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,1023,0.08592533071835835
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,1023,0.050330668687820435
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,63,0.01748266691962878
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,3,0.009296000003814697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,3,0.009296000003814697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,7,0.008992000172535578
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,7,0.008938666433095932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,127,0.017263999829689663
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,15,0.00903466654320558
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,31,0.009082666908701261
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,4095,0.3041653235753377
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,4095,0.16360533237457275
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,31,0.009637333452701569
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,2047,0.16103999813397726
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,1,0.009317333499590555
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,63,0.009258666386206945
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,255,0.00914666677514712
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,1,0.009098666409651438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,2047,0.08913600444793701
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,63,0.009008000294367472
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,255,0.00895999992887179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,511,0.009152000149091085
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,511,0.011120000233252844
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,1023,0.008965333302815756
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,2047,0.00955200009047985
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,2047,0.010538666198650995
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,4095,0.013002666334311167
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,4095,0.012970666090647379
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,8191,0.015210667004187902
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,8191,0.015850666910409927
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,16383,0.017024000485738117
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,32767,0.03355200091997782
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,32767,0.019152000546455383
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,65535,0.0518453319867452
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,65535,0.035487999518712364
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,131071,0.08916800220807393
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,131071,0.053930665055910744
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,15,0.009061333412925402
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,1,0.008943999807039896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,1,0.009226666763424873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,127,0.009253333633144697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,3,0.009328000247478485
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,127,0.009317333499590555
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,3,0.00943999985853831
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,7,0.008954666554927826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,7,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,1023,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,15,0.00926399976015091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,15,0.00903466654320558
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,31,0.008938666433095932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,31,0.009535999968647957
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,63,0.008922666932145754
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,63,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,127,0.009450666606426239
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,255,0.009029333169261614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,255,0.009226666763424873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,511,0.01101333275437355
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,511,0.01101333275437355
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,1023,0.011034666250149408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,1023,0.011018666128317514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,2047,0.012997332960367203
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,2047,0.013034666577974955
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,4095,0.014949332922697067
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,8191,0.01714133347074191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,8191,0.015072000523408255
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,16383,0.017509333789348602
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,16383,0.03286399940649668
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,32767,0.050517335534095764
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,32767,0.033514666060606636
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,65535,0.0881226658821106
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,16383,0.016602666427691776
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,65535,0.05190399785836538
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,131071,0.09091732899347942
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,131071,0.16062933206558228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,127,0.008954666554927826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,4095,0.013482666263977686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,1,0.017114666601022083
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,1,0.01922133316596349
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,3,0.01740266631046931
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,3,0.019359999646743137
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,7,0.017136000096797943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,7,0.01929066702723503
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,15,0.019253333409627277
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,15,0.01748266691962878
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,31,0.019365333020687103
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,31,0.01710933322707812
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,63,0.019152000546455383
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,63,0.017258666455745697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,127,0.01729600007335345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,127,0.01951466624935468
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,255,0.03044266750415166
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,255,0.017456000049908955
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,511,0.04911999901135763
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,511,0.03134933362404505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,1,0.03156266609827677
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,1023,0.0867199997107188
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,1,0.02749866743882497
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,1023,0.05044800043106079
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,2047,0.08962133526802063
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,2047,0.1593226691087087
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,3,0.03140799949566523
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,7,0.03047466774781545
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,15,0.03141333411137263
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,7,0.02739199995994568
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,15,0.027653334041436512
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,3,0.02737066646416982
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,63,0.03145066648721695
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,63,0.027664000789324444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,127,0.035605333745479584
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,127,0.027813332776228588
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,31,0.03141333411137263
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,31,0.02758399893840154
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,255,0.03555200000603994
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,255,0.05072000126043955
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,511,0.08740267157554626
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,511,0.054085334142049156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,1023,0.16051733493804932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,1023,0.09087466200192769
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,1,0.03141333411137263
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,1,0.02752533306678136
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,2047,0.30620266993840534
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,2047,0.16607466340065002
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,3,0.03143466760714849
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,3,0.02787200113137563
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,15,0.031504000226656594
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,7,0.02779199928045273
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,7,0.03156266609827677
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,15,0.02758399893840154
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,63,0.031397332747777305
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,31,0.031317333380381264
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,31,0.02796799937884013
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,63,0.027349332968393963
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,127,0.02805333336194356
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,127,0.035562666753927864
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,255,0.05044800043106079
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,511,0.08846933643023173
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,255,0.03612266729275385
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,1,0.04578666885693868
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,511,0.053930665055910744
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,3,0.056015998125076294
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,1023,0.09153599540392558
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,1023,0.1628213326136271
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,1,0.056101332108179726
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,3,0.04655999938646952
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,7,0.055589333176612854
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,7,0.045824001232783
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,15,0.04600533346335093
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,15,0.054805333415667214
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,31,0.045935998360315956
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,63,0.05618133147557577
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,31,0.05615466833114624
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,63,0.046021332343419395
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,127,0.05960000058015188
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,127,0.05089066425959269
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,1,0.009194666519761086
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,3,0.01403733342885971
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,3,0.009183999771873156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,7,0.009162666896979014
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,1,0.009189333145817121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,7,0.009189333145817121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,15,0.008938666433095932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,255,0.09090666969617207
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,15,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,31,0.00897066667675972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,31,0.009183999771873156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,63,0.008890666688481966
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,255,0.05990933378537496
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,127,0.009056000038981438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,127,0.009189333145817121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,255,0.009136000027259191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,511,0.008992000172535578
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,511,0.01110400011142095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,1023,0.00927466650803884
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,1023,0.011194666226704916
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,2047,0.012981332838535309
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,2047,0.01128000020980835
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,4095,0.015040000279744467
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,4095,0.013386666774749756
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,8191,0.017055999487638474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,8191,0.015413332730531693
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,63,0.008986666798591614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,16383,0.03205333401759466
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,16383,0.017317333569129307
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,32767,0.05133866767088572
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,32767,0.033546666304270424
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,65535,0.08833066622416179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,65535,0.05186666548252106
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,255,0.009381333366036415
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,1,0.009285333255926767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,1,0.009056000038981438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,3,0.009029333169261614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,131071,0.16051733493804932
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,3,0.009445333232482275
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,7,0.009039999917149544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,7,0.009002666920423508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,15,0.00933333362142245
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,15,0.009328000247478485
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,31,0.00922133338948091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,31,0.009663999701539675
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,63,0.00922133338948091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,63,0.008912000184257826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,127,0.009904000287254652
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,127,0.009269333134094873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,511,0.16427200039227804
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,255,0.009663999701539675
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,255,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,511,0.01098666712641716
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,511,0.010346666599313417
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,1023,0.011253333340088526
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,511,0.09286399682362874
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,2047,0.013061333447694778
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,2047,0.011018666128317514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,4095,0.015381333728631338
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,4095,0.013338666409254074
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,8191,0.029706666866938274
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,16383,0.04798933366934458
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,16383,0.03148266673088074
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,32767,0.08501866459846497
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,32767,0.05009600023428599
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,65535,0.15868799885114035
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,131071,0.0909546713034312
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,65535,0.08835732936859131
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,1,0.009018666421373686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,1,0.009695999945203463
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,131071,0.3060213327407837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,3,0.009109333157539368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,131071,0.16354667147000632
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,1023,0.010933333386977514
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,7,0.009136000027259191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,3,0.009312000125646591
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,7,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,15,0.009328000247478485
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,15,0.009098666409651438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,8191,0.015322666615247726
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,31,0.009242666885256767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,63,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,63,0.009226666763424873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,127,0.008933333059151968
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,127,0.009312000125646591
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,255,0.009226666763424873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,255,0.009695999945203463
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,511,0.009578666960199675
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,511,0.011061333119869232
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,1023,0.01101333275437355
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,1023,0.011258666714032492
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,2047,0.013066666821638743
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,2047,0.011328000575304031
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,4095,0.014901333798964819
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,4095,0.013306666165590286
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,8191,0.030000001192092896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,16383,0.048842668533325195
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,1023,0.16615466276804605
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,1023,0.30806400378545123
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,16383,0.03176533430814743
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,32767,0.08709866801897685
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,32767,0.05193066596984863
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,65535,0.16076800227165222
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,1,0.009573333586255709
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,3,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,3,0.009290666629870733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,7,0.009354666496316591
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,7,0.009173333023985228
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,15,0.009136000027259191
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,15,0.00921066664159298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,31,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,31,0.009189333145817121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,63,0.008986666798591614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,63,0.009322666873534521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,8191,0.01544533297419548
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,127,0.009008000294367472
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,255,0.009557333464423815
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,255,0.009130666653315226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,511,0.009493333597977957
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,1023,0.009434666484594345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,1023,0.009141333401203156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,31,0.009056000038981438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,2047,0.009599999835093817
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,2047,0.009839999799927076
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,4095,0.011477333803971609
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,8191,0.015226667126019796
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,65535,0.09019200007120769
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,8191,0.015450666348139444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,131071,0.16482667128245035
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,16383,0.01729600007335345
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,16383,0.019359999646743137
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,127,0.00914666677514712
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,32767,0.027482666075229645
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,65535,0.03175999969244003
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,65535,0.032287999987602234
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,131071,0.03388266762097677
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,32767,0.026586666703224182
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,131071,0.03162133445342382
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,1,0.009216000015536943
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,1,0.008965333302815756
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,3,0.009008000294367472
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,3,0.00916800027092298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,7,0.00916800027092298
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,7,0.008879999940594038
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,15,0.00895999992887179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,31,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,31,0.008901333436369896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,63,0.008965333302815756
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,131071,0.3065066734949748
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,63,0.009226666763424873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,127,0.008869333192706108
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,127,0.008986666798591614
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,255,0.009178666397929192
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,511,0.009232000137368837
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,255,0.00927466650803884
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,1023,0.009178666397929192
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,1023,0.009018666421373686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,2047,0.009285333255926767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,2047,0.009088000282645226
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,4095,0.010981333752473196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,4095,0.010970667004585266
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,8191,0.015114666273196539
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,4095,0.01116266722480456
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,8191,0.015109332899252573
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,16383,0.019141333798567455
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,16383,0.01937599976857503
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,32767,0.0230880007147789
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,32767,0.021231998999913532
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,65535,0.025306666890780132
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,65535,0.023530667026837666
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,131071,0.041984001795450844
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,131071,0.02535466601451238
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,1,0.00985599992175897
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,1,0.010853332777818045
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,3,0.009189333145817121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,3,0.01090666651725769
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,1,0.009103999783595404
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,7,0.009258666386206945
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,7,0.009285333255926767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,511,0.011306667079528173
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,15,0.009578666960199675
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,15,0.010874666273593903
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,15,0.008976000050703684
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,31,0.009290666629870733
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,31,0.009285333255926767
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,63,0.00979200005531311
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,63,0.010981333752473196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,127,0.010821333775917688
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,255,0.009002666920423508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,255,0.009573333586255709
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,511,0.010512000570694605
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,511,0.01097600037852923
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,511,0.010826667149861654
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,1023,0.013125333935022354
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,1023,0.011050666371981302
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,2047,0.014538666854302088
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,2047,0.013072000195582708
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,4095,0.031199999153614044
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,4095,0.01624533285697301
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,8191,0.04832000037034353
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,8191,0.031557333966096245
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,16383,0.0867680013179779
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,16383,0.05031466484069824
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,1,0.009317333499590555
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,1,0.008943999807039896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,65535,0.3089546759923299
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,32767,0.16106133659680685
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,3,0.008997333546479544
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,3,0.009226666763424873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,7,0.009061333412925402
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,65535,0.1622933348019918
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,32767,0.08804800113042195
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,15,0.011871999750534693
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,15,0.009343999748428663
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,63,0.008954666554927826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,31,0.009994666402538618
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,31,0.009056000038981438
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,127,0.008954666554927826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,131071,0.6027253468831381
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,255,0.009045333291093508
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,511,0.011061333119869232
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,1023,0.00915733352303505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,511,0.009546666716535887
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,1023,0.011071999867757162
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,2047,0.01099733387430509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,2047,0.010981333752473196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,4095,0.011039999624093374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,8191,0.015381333728631338
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,8191,0.015018666783968607
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,16383,0.0174346665541331
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,7,0.009248000259200731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,16383,0.017263999829689663
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,127,0.009018666421373686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,32767,0.02022933339079221
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,32767,0.019199999670187633
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,63,0.009269333134094873
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,65535,0.021231998999913532
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,65535,0.03775466730197271
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,131071,0.05421333511670431
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,131071,0.03761066744724909
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,127,0.00890666681031386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,131071,0.31090666850407916
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,255,0.00891733355820179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,1,0.011029332876205444
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,1,0.011157333850860596
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,3,0.010874666273593903
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,3,0.010703999549150467
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,7,0.011055999745925268
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,15,0.010885333021481832
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,15,0.010949333508809408
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,31,0.011152000476916632
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,7,0.01099733387430509
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,63,0.010928000013033548
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,63,0.010821333775917688
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,127,0.010954666882753372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,127,0.010928000013033548
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,255,0.010960000256697336
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,255,0.01118933285276095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,511,0.011221333096424738
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,511,0.011391999820868174
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,1023,0.013248000293970108
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,1023,0.013007999708255133
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,2047,0.02975466599067052
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,2047,0.015168000012636185
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,4095,0.048394665122032166
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,8191,0.08687466382980347
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,4095,0.03156800071398417
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,8191,0.04990933338801066
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,31,0.011077333241701126
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,4095,0.010981333752473196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,16383,0.16037866473197937
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,32767,0.16225066781044006
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,16383,0.0867733359336853
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,1,0.011061333119869232
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,32767,0.30688534180323285
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,1,0.01110400011142095
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,65535,0.31108800570170086
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,3,0.010944000134865442
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,7,0.011285333583752314
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,65535,0.6041973431905111
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,7,0.011226666470368704
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,15,0.012853333105643591
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,31,0.01179733375708262
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,31,0.01128000020980835
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,15,0.011450666934251785
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,63,0.011066666493813196
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,127,0.012746666868527731
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,63,0.011418666690587997
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,127,0.011141333729028702
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,255,0.012741333494583765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,255,0.01126933346192042
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,3,0.011077333241701126
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,511,0.015157333264748255
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,511,0.01303999995191892
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,1023,0.029418667157491047
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,1023,0.01609066625436147
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,2047,0.04955733319123586
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,2047,0.031541332602500916
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,8191,0.16062399744987488
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,8191,0.08706667025883992
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,16383,0.3106773296991984
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,4095,0.08676266670227051
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,4095,0.05023466547330221
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,16383,0.1639253298441569
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,1,0.01516266663869222
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,3,0.015178666760524115
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,1,0.01328533391157786
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,3,0.01339200014869372
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,7,0.015013333410024643
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,15,0.013050666699806849
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,31,0.015141333142916361
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,31,0.013125333935022354
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,7,0.013733333597580591
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,63,0.015109332899252573
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,15,0.014991999914248785
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,63,0.013397333522637686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,127,0.014837333311637243
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,127,0.013248000293970108
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,255,0.015109332899252573
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,255,0.012986666212479273
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,511,0.03035199890534083
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,511,0.01728533332546552
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,1023,0.03180266668399175
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,1023,0.04961599906285604
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,2047,0.08714133501052856
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,1,0.00890666681031386
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,1,0.00897066667675972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,3,0.009178666397929192
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,7,0.00915733352303505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,7,0.009183999771873156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,2047,0.05092266698678335
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,15,0.00898133342464765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,15,0.009109333157539368
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,31,0.00901333304742972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,63,0.008976000050703684
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,63,0.00895999992887179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,127,0.009279999881982803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,127,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,255,0.009183999771873156
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,8191,0.31251732508341473
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,255,0.00926399976015091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,511,0.009365333244204521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,1023,0.009888000165422758
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,1023,0.011674666156371435
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,2047,0.011039999624093374
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,2047,0.011146667102972666
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,511,0.00926399976015091
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,4095,0.013573333621025085
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,4095,0.013130666067202887
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,8191,0.015306666493415833
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,8191,0.014997333288192749
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,16383,0.015658666690190632
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,3,0.00901333304742972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,31,0.00901333304742972
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,32767,0.03347733368476232
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,32767,0.019141333798567455
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,65535,0.05229333539803823
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,65535,0.03555733213822047
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,4095,0.16395733753840128
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,131071,0.05433600147565206
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,4095,0.08709333340326945
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,8191,0.16361600160598755
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,16383,0.017386666188637417
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,131071,0.08876799543698628
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,1,0.01754133279124896
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,1,0.019306667149066925
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,3,0.01711999997496605
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,3,0.01952533299724261
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,7,0.017344000438849132
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,7,0.01932799940307935
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,15,0.019253333409627277
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,15,0.017344000438849132
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,31,0.019434666881958645
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,31,0.017488000293572743
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,63,0.019199999670187633
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,63,0.01710933322707812
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,127,0.019194666296243668
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,127,0.017551999539136887
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,255,0.03010133405526479
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,255,0.017429333180189133
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,511,0.031397332747777305
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,511,0.04996266464392344
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,1023,0.0860586663087209
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,1023,0.05106133222579956
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,2047,0.16038933396339417
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,2047,0.08913600444793701
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,1,0.03156800071398417
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,1,0.027514666318893433
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,4095,0.16453333695729574
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,4095,0.3083253304163615
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,3,0.03150933235883713
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,3,0.027456000447273254
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,15,0.03155199935038885
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,15,0.027503999571005504
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,7,0.027717334528764088
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,7,0.03127466638882955
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,63,0.03143999973932902
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,63,0.02756800005833308
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,127,0.0364533339937528
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,31,0.03158933420976003
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,31,0.02740799884001414
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,127,0.027514666318893433
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,255,0.051088000337282814
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,255,0.03541333228349686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,1,0.008954666554927826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,3,0.009189333145817121
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,3,0.008949333180983862
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,511,0.08827733000119527
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,7,0.009322666873534521
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,1,0.009050666665037474
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,15,0.00919999989370505
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,31,0.008837333569924036
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,31,0.00891733355820179
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,63,0.009008000294367472
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,511,0.05347733199596405
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,63,0.008885333314538002
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,127,0.008954666554927826
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,255,0.00902399979531765
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,255,0.009253333633144697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,7,0.008992000172535578
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,511,0.010960000256697336
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,511,0.01090666651725769
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,1023,0.01102399950226148
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,1023,0.010885333021481832
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,2047,0.0129120002190272
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,15,0.009018666421373686
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,2047,0.011962667107582092
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,4095,0.013445333888133367
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,8191,0.017173333714405697
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,8191,0.01534933348496755
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,127,0.009237333511312803
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,16383,0.03181333343187968
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,16383,0.017386666188637417
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,32767,0.05179733534653982
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,32767,0.03369600077470144
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,65535,0.09116266171137492
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,65535,0.0525493323802948
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,131071,0.16486400365829468
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,131071,0.09158933162689209
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,1023,0.16150400042533875
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,4095,0.015002666662136713
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,1023,0.09077866872151692
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,2047,0.3080853422482808
VLLM,0.14.1,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,2047,0.1644266645113627
