framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,1,0.008762666955590248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,31,0.008693333094318708
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,31,0.008693333094318708
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,63,0.008639999975760778
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,63,0.008309333274761835
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,127,0.008623999853928884
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,127,0.008576000109314919
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,255,0.008565333361426989
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,255,0.007685333490371704
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,511,0.009349333122372627
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,511,0.0086666668454806
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,1023,0.011733333269755045
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,1023,0.010645333677530289
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,2047,0.027301333844661713
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,2047,0.013077333569526672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,4095,0.04473066826661428
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,4095,0.027061333258946735
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,1,0.008922666932145754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,1,0.008639999975760778
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,3,0.008799999952316284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,3,0.008645333349704742
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,7,0.009226666763424873
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,7,0.00874133345981439
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,15,0.008879999940594038
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,15,0.00873066671192646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,31,0.008976000050703684
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,31,0.008581333483258883
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,63,0.008912000184257826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,63,0.008656000097592672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,127,0.008879999940594038
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,15,0.008576000109314919
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,127,0.00877333308259646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,7,0.008592000231146812
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,255,0.008799999952316284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,7,0.00877333308259646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,255,0.009018666421373686
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,511,0.01209066684047381
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,511,0.010773333410422007
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,1023,0.0264533335963885
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,1023,0.012944000462690989
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,2047,0.027093333502610523
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,4095,0.04553066690762838
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,4095,0.08063466846942902
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,3,0.008938666433095932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,1,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,1,0.006720000257094701
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,3,0.007391999786098798
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,15,0.009178666397929192
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,7,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,3,0.007029333462317784
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,7,0.006858666737874349
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,15,0.007242666557431221
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,31,0.00697066696981589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,15,0.0068693334857622785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,31,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,63,0.006981333096822103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,63,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,1,0.0075519997626543045
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,127,0.006810666372378667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,127,0.0069226666043202085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,255,0.006640000268816948
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,255,0.00701333334048589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,511,0.008912000184257826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,3,0.008933333059151968
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,1023,0.00898133342464765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,511,0.008821333448092142
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,2047,0.008639999975760778
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,2047,0.008757333581646284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,8191,0.010687999427318573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,4095,0.008933333059151968
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,8191,0.010037333394090334
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,16383,0.014778666198253632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,1,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,1,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,3,0.006655999769767125
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,3,0.006778666749596596
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,7,0.006720000257094701
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,7,0.00684799998998642
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,15,0.006810666372378667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,15,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,31,0.006640000268816948
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,31,0.006831999868154526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,63,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,127,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,127,0.006730666384100914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,255,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,255,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,511,0.008693333094318708
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,2047,0.0454720010360082
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,511,0.006981333096822103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,1023,0.008752000207702318
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,1023,0.008634666601816813
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,2047,0.008767999708652496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,2047,0.00874133345981439
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,4095,0.00884799969693025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,4095,0.00878399983048439
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,8191,0.011359999577204386
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,16383,0.015178666760524115
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,1,0.006773333375652631
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,1,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,3,0.00684799998998642
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,4095,0.00871999996403853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,3,0.006581333155433337
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,16383,0.014815999815861383
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,7,0.006597333277265231
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,15,0.006858666737874349
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,15,0.006810666372378667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,31,0.0068693334857622785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,63,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,63,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,63,0.006746666505932808
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,127,0.006704000135262807
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,127,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,255,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,511,0.008869333192706108
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,511,0.008821333448092142
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,1023,0.009045333291093508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,8191,0.010714666297038397
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,16383,0.012970666090647379
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,1023,0.00903466654320558
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,2047,0.008965333302815756
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,2047,0.008869333192706108
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,4095,0.008912000184257826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,1023,0.008938666433095932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,4095,0.008789333204428354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,8191,0.013082666943470636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,8191,0.013013333082199097
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,16383,0.014933332800865173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,16383,0.016490666816631954
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,7,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,1,0.006709333509206772
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,3,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,3,0.006538666784763336
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,7,0.006768000001708667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,7,0.00684799998998642
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,15,0.006837333242098491
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,31,0.006581333155433337
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,15,0.0069226666043202085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,31,0.006789333497484525
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,31,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,63,0.006698666761318843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,63,0.006831999868154526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,127,0.006640000268816948
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,127,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,255,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,255,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,511,0.008639999975760778
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,511,0.007178666690985362
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,1023,0.008858666444818178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,1023,0.008703999842206636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,2047,0.008602666358153025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,2047,0.008874666566650072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,4095,0.010735999792814255
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,4095,0.011034666250149408
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,8191,0.013989333063364029
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,16383,0.014842666685581207
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,16383,0.035445332527160645
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,1,0.010816000401973724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,1,0.008949333180983862
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,3,0.010757333288590113
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,3,0.008938666433095932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,7,0.010890666395425797
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,7,0.009098666409651438
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,15,0.009162666896979014
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,31,0.010751999914646149
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,31,0.009077333534757296
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,63,0.010965333630641302
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,63,0.00877333308259646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,127,0.010698666175206503
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,127,0.00915733352303505
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,1,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,255,0.010981333752473196
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,255,0.00874133345981439
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,511,0.012719999998807907
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,511,0.011183999478816986
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,2047,0.045519997676213585
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,1023,0.014997333288192749
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,1023,0.027109332382678986
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,2047,0.027808000644048054
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,1,0.012586666891972223
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,1,0.010608000059922537
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,255,0.0069386667261521024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,7,0.010863999525705973
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,3,0.011066666493813196
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,15,0.012800000607967377
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,15,0.010650667051474253
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,31,0.012837332983811697
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,15,0.010661333799362183
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,63,0.012698666503032049
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,63,0.010682666053374609
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,127,0.011424000064531961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,127,0.012869333227475485
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,255,0.0129120002190272
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,255,0.010725333044926325
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,511,0.013072000195582708
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,511,0.027136000494162243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,1023,0.045567999283472695
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,1023,0.02739733209212621
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,1,0.007040000210205714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,1,0.0069866664707660675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,3,0.0069973332186539965
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,2047,0.04610666632652283
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,3,0.0069973332186539965
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,2047,0.08270399769147237
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,7,0.011962667107582092
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,7,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,15,0.00697066696981589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,15,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,3,0.012714666624863943
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,31,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,63,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,63,0.007157333195209503
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,127,0.00684799998998642
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,127,0.0069866664707660675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,255,0.006831999868154526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,255,0.006677333265542984
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,511,0.00879466657837232
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,511,0.00879466657837232
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,1023,0.008997333546479544
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,1023,0.008938666433095932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,2047,0.008885333314538002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,2047,0.009008000294367472
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,4095,0.01201066623131434
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,8191,0.01509333277742068
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,8191,0.013157332936922709
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,16383,0.029440000653266907
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,16383,0.016842667013406754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,1,0.007114666824539502
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,1,0.006682666639486949
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,3,0.006805333619316419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,3,0.006704000135262807
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,7,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,15,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,8191,0.012106666962305704
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,15,0.006528000036875407
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,31,0.00702400008837382
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,31,0.00707733320693175
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,63,0.007045333584149678
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,31,0.011050666371981302
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,127,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,127,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,255,0.00707733320693175
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,255,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,511,0.008826666822036108
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,511,0.007125333572427432
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,4095,0.010954666882753372
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,1023,0.008879999940594038
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,1023,0.00874133345981439
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,2047,0.009008000294367472
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,4095,0.01268799975514412
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,4095,0.010879999647537867
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,8191,0.036490666369597115
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,8191,0.012741333494583765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,16383,0.05550399919350942
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,16383,0.027242665489514668
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,7,0.00785600021481514
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,1,0.013786666095256805
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,7,0.006805333619316419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,3,0.01470400020480156
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,31,0.007002666592597961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,3,0.012703999876976013
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,7,0.014778666198253632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,7,0.012671999633312225
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,15,0.015082667271296183
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,63,0.0064853330453236895
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,15,0.012826666235923767
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,31,0.015013333410024643
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,31,0.012890666723251343
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,63,0.014874666929244995
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,63,0.012965332716703415
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,127,0.014874666929244995
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,127,0.012757333616415659
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,2047,0.010768000036478043
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,255,0.015008000036080679
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,255,0.012736000120639801
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,511,0.02775466690460841
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,511,0.016597333053747814
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,1,0.017103999853134155
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,1,0.015893333901961643
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,1,0.014192000031471252
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,1023,0.04619733492533366
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,3,0.01695999999841054
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,1023,0.02906133234500885
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,3,0.014815999815861383
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,7,0.015119999647140503
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,15,0.016970666746298473
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,15,0.015002666662136713
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,7,0.01716800034046173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,31,0.014938666174809137
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,63,0.014773332824309668
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,127,0.017157333592573803
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,127,0.015040000279744467
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,255,0.034586665530999504
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,255,0.015168000012636185
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,63,0.01676799977819125
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,31,0.016927999754746754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,1023,0.08162666857242584
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,511,0.02737066646416982
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,511,0.056218668818473816
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,1,0.022474666436513264
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,1,0.018063999712467194
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,3,0.021231998999913532
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,3,0.01699200024207433
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,7,0.021173333128293354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,15,0.017258666455745697
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,7,0.017157333592573803
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,15,0.022783999641736347
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,31,0.021338666478792827
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,31,0.016943999876578648
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,63,0.021568000316619873
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,63,0.01720533271630605
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,1023,0.04722133278846741
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,127,0.021429332594076794
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,127,0.01708799973130226
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,255,0.028010666370391846
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,255,0.017082666357358296
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,1,0.02735466758410136
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,1,0.023103999594847362
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,3,0.02739199995994568
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,3,0.023237332701683044
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,15,0.023311999936898548
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,7,0.02741866558790207
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,7,0.023247999449570973
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,15,0.02739199995994568
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,31,0.02734400083621343
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,31,0.02332799881696701
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,63,0.027077332139015198
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,63,0.02311466634273529
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,127,0.02294933299223582
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,127,0.04085866610209147
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,255,0.02884799987077713
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,255,0.05376533170541128
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,1,0.03766400118668874
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,1,0.027301333844661713
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,3,0.0376800000667572
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,3,0.027509334186712902
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,7,0.03730133424202601
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,7,0.02743999908367793
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,31,0.03759466608365377
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,31,0.0272533322374026
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,15,0.0271519993742307
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,15,0.03732266773780187
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,63,0.03739733248949051
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,63,0.027306665976842243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,127,0.027050666511058807
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,127,0.03764266769091288
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,1,0.04991999765237173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,1,0.039546666045983635
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,3,0.039818666875362396
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,7,0.04919999837875366
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,7,0.03979733337958654
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,15,0.04896000027656555
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,15,0.039701332648595176
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,31,0.0496373325586319
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,1,0.0069866664707660675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,1,0.00702400008837382
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,63,0.05202133456865946
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,63,0.03953066716591517
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,3,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,3,0.007130666946371396
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,15,0.007098666702707608
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,15,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,7,0.0069973332186539965
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,7,0.00696000022192796
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,31,0.0069226666043202085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,31,0.006730666384100914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,63,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,63,0.006682666639486949
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,3,0.0496319979429245
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,127,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,127,0.00702400008837382
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,127,0.051674668987592064
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,255,0.00697066696981589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,255,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,511,0.008885333314538002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,511,0.008832000195980072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,1023,0.009072000160813332
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,1023,0.008602666358153025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,4095,0.012810666114091873
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,4095,0.011050666371981302
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,127,0.04074666649103165
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,8191,0.027056001126766205
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,8191,0.01479999969402949
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,16383,0.028175999720891316
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,2047,0.01110400011142095
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,1,0.006666666517655055
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,3,0.0069386667261521024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,3,0.006735999758044879
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,7,0.007135999699433644
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,7,0.006693333387374878
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,15,0.0069386667261521024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,15,0.00666133314371109
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,31,0.00697066696981589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,31,0.006810666372378667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,63,0.006645333642760913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,63,0.0069386667261521024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,127,0.0069226666043202085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,255,0.006714666883150737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,255,0.0069919998447100324
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,511,0.008858666444818178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,511,0.008367999767263731
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,1023,0.010064000263810158
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,1023,0.008709333216150602
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,2047,0.011168000598748526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,2047,0.010842667271693548
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,4095,0.036517334481080375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,4095,0.012698666503032049
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,2047,0.009472000102202097
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,8191,0.05570666491985321
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,8191,0.027301333844661713
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,16383,0.04550399879614512
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,16383,0.08540266752243042
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,16383,0.04554666578769684
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,1,0.007034666836261749
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,31,0.03975466638803482
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,127,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,float16,1,0.06831466654936473
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,1,0.04748799900213877
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,float16,3,0.06828799843788147
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,3,0.04786133269468943
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,float16,7,0.06830933193365733
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,7,0.04780800143877665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,float16,15,0.068122665087382
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,15,0.047450666626294456
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,float16,31,0.0680320014556249
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,31,0.04763199885686239
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,float16,63,0.06825600067774455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,63,0.0476800004641215
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,float16,1,0.08899733424186707
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,1,0.07298666735490163
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,float16,3,0.08970133463541667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,float16,7,0.08955732981363933
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,3,0.07239999870459239
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,7,0.07438933352629344
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,float16,15,0.08916800220807393
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,15,0.07396266857783
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,float16,31,0.08912000060081482
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,31,0.07254933317502339
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,float16,63,0.08961600065231323
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,63,0.07246399919191997
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,float16,1,0.1276800036430359
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,1,0.0867733359336853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,float16,3,0.12970133622487387
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,3,0.08784533540407817
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,float16,7,0.12921599547068277
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,7,0.08854400118192036
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,float16,15,0.12995200355847678
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,15,0.08794132868448894
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,float16,1,0.16875733931859335
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,1,0.13768000404040018
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,float16,31,0.12874666849772134
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,float16,3,0.16875199476877847
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,31,0.08689066767692566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,1,0.00879466657837232
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,1,0.00702400008837382
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,3,0.1378720005353292
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,3,0.008762666955590248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,float16,7,0.16907199223836264
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,3,0.00706666645904382
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,7,0.007311999797821045
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,15,0.00867733359336853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,7,0.00702400008837382
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,15,0.0068853336075941724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,31,0.008714666590094566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,31,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,63,0.007311999797821045
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,63,0.00697066696981589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,127,0.00706666645904382
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,127,0.0069226666043202085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,255,0.008661333471536636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,255,0.006981333096822103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,511,0.008842666943868002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,7,0.13768000404040018
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,511,0.00873066671192646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,1023,0.010687999427318573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,1023,0.00919999989370505
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,2047,0.012762666990359625
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,2047,0.010762666662534079
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,4095,0.015018666783968607
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,4095,0.02735466758410136
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,8191,0.04541333516438802
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,1,0.008207999790708223
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,1,0.007018666714429855
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,3,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,3,0.008074666683872541
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,7,0.007957333077987036
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,7,0.006805333619316419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,15,0.007354666789372762
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,15,0.006837333242098491
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,31,0.008277333031098047
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,31,0.006981333096822103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,63,0.008352000266313553
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,63,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,127,0.008112000301480293
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,255,0.009061333412925402
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,255,0.006746666505932808
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,511,0.008954666554927826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,float16,15,0.1686453421910604
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,15,0.13770133256912231
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,1023,0.010874666273593903
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,1023,0.008703999842206636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,31,0.13788800438245138
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,2047,0.03540800015131632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,8191,0.028069332242012024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,2047,0.012837332983811697
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,float16,31,0.1683680017789205
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,4095,0.054986665646235146
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,8191,0.04548799991607666
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,1,0.008885333314538002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,3,0.00874133345981439
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,3,0.008229333286484083
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,8191,0.0807360013326009
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,7,0.008714666590094566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,1,0.008863999818762144
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,7,0.007424000029762586
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,15,0.008693333094318708
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,127,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,15,0.0069866664707660675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,31,0.008922666932145754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,31,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,63,0.008682666967312494
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,63,0.007045333584149678
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,127,0.008634666601816813
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,127,0.008645333349704742
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,255,0.00879466657837232
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,511,0.008837333569924036
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,511,0.0086666668454806
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,1023,0.01166933278242747
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,1023,0.010911999891201654
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,2047,0.02698666602373123
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,1,0.009056000038981438
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,1,0.008778666456540426
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,4095,0.045007998744646706
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,4095,0.02741333345572154
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,4095,0.027114666998386383
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,3,0.00860799973209699
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,7,0.00884799969693025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,7,0.009029333169261614
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,15,0.008698666468262672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,15,0.008832000195980072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,31,0.009045333291093508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,31,0.008863999818762144
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,63,0.009050666665037474
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,127,0.00901333304742972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,127,0.008943999807039896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,255,0.008943999807039896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,255,0.008842666943868002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,511,0.011658667276302973
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,511,0.010832000523805618
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,255,0.007040000210205714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,1023,0.026314665873845417
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,511,0.008965333302815756
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,1023,0.0129120002190272
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,1,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,2047,0.027162666122118633
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,2047,0.04478399952252706
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,2047,0.013114667187134424
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,4095,0.080485333998998
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,4095,0.04578666885693868
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,3,0.006789333497484525
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,7,0.007002666592597961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,3,0.007040000210205714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,7,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,15,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,31,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,31,0.007055999711155891
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,63,0.007167999943097432
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,63,0.006789333497484525
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,127,0.00696000022192796
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,127,0.007061333085099856
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,3,0.00897066667675972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,511,0.008602666358153025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,511,0.00879466657837232
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,1023,0.009088000282645226
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,1023,0.00892800030608972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,2047,0.008821333448092142
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,2047,0.00892800030608972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,4095,0.008714666590094566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,4095,0.008762666955590248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,8191,0.010778666784365972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,8191,0.009653333574533463
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,63,0.008752000207702318
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,16383,0.014864000181357065
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,16383,0.0144213338692983
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,1,0.006735999758044879
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,1,0.00684799998998642
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,3,0.006597333277265231
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,3,0.007082666580875714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,7,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,7,0.006570666407545407
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,15,0.006757333253820737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,15,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,31,0.0069226666043202085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,63,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,127,0.0069919998447100324
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,127,0.00655466690659523
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,255,0.006677333265542984
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,255,0.006624000146985054
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,1,0.0069919998447100324
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,511,0.008757333581646284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,15,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,1023,0.008821333448092142
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,1023,0.008698666468262672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,255,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,255,0.007050666958093643
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,2047,0.008613333106040955
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,4095,0.009008000294367472
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,8191,0.011850666254758835
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,8191,0.010890666395425797
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,16383,0.015061333775520325
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,16383,0.013034666577974955
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,1,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,1,0.007151999821265538
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,3,0.006784000123540561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,3,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,7,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,7,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,15,0.0069973332186539965
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,31,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,15,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,63,0.007109333450595538
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,31,0.007029333462317784
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,31,0.0069973332186539965
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,63,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,63,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,127,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,127,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,255,0.007141333073377609
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,255,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,511,0.008986666798591614
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,1023,0.00914666677514712
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,1023,0.00890666681031386
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,2047,0.008810666700204214
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,2047,0.008714666590094566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,4095,0.009077333534757296
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,4095,0.009029333169261614
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,511,0.00701333334048589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,8191,0.012784000486135483
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,8191,0.012869333227475485
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,16383,0.015178666760524115
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,16383,0.015008000036080679
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,1,0.006773333375652631
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,1,0.006714666883150737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,3,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,2047,0.00891733355820179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,3,0.006640000268816948
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,7,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,7,0.006757333253820737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,15,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,31,0.007018666714429855
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,4095,0.008645333349704742
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,63,0.006805333619316419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,127,0.007087999954819679
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,255,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,255,0.0068853336075941724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,511,0.008703999842206636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,511,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,1023,0.008885333314538002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,1023,0.008656000097592672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,2047,0.008725333337982496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,2047,0.008832000195980072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,4095,0.010965333630641302
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,4095,0.01073066641887029
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,8191,0.01504533365368843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,8191,0.012682666381200155
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,16383,0.03864533454179764
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,16383,0.015360000232855478
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,1,0.010714666297038397
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,1,0.008778666456540426
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,3,0.010901333143313726
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,511,0.008725333337982496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,3,0.009008000294367472
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,7,0.01099733387430509
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,7,0.008885333314538002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,15,0.010874666273593903
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,15,0.008922666932145754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,31,0.01073066641887029
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,31,0.00891733355820179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,15,0.006725333631038666
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,63,0.010794666906197866
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,63,0.009061333412925402
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,63,0.00666133314371109
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,127,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,127,0.008885333314538002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,255,0.010698666175206503
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,255,0.009103999783595404
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,511,0.013050666699806849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,511,0.011445333560307821
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,1023,0.02719466636578242
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,2047,0.04554133117198944
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,2047,0.027808000644048054
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,1,0.012736000120639801
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,1,0.010805333654085795
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,3,0.010709332923094431
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,7,0.011391999820868174
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,3,0.012576000144084295
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,7,0.010640000303586325
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,15,0.012869333227475485
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,15,0.010901333143313726
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,31,0.011477333803971609
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,31,0.01081066702802976
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,63,0.011989332735538483
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,63,0.010826667149861654
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,127,0.012815999488035837
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,127,0.0107893335322539
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,255,0.012144000579913458
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,255,0.010677333921194077
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,31,0.006735999758044879
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,127,0.0107893335322539
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,511,0.03526933242877325
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,511,0.013167999684810638
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,1023,0.02870933214823405
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,1023,0.04543466866016388
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,2047,0.08264533181985219
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,3,0.006613333399097125
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,2047,0.04615999758243561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,7,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,1,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,1023,0.014794666320085526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,7,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,15,0.007007999966541926
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,31,0.007125333572427432
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,63,0.006826666494210561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,63,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,31,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,127,0.007045333584149678
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,127,0.0069866664707660675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,255,0.007029333462317784
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,255,0.0068693334857622785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,511,0.008752000207702318
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,511,0.008687999720374743
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,1023,0.009088000282645226
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,2047,0.008912000184257826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,1023,0.008752000207702318
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,4095,0.012053333222866058
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,4095,0.011066666493813196
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,8191,0.015189333508412043
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,16383,0.02940266579389572
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,16383,0.016864000509182613
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,1,0.0068693334857622785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,1,0.006837333242098491
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,3,0.00701333334048589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,3,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,7,0.006810666372378667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,7,0.006666666517655055
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,15,0.006826666494210561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,15,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,31,0.006666666517655055
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,31,0.006831999868154526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,63,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,63,0.006640000268816948
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,127,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,127,0.00660800002515316
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,3,0.007034666836261749
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,255,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,255,0.00660800002515316
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,511,0.008682666967312494
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,1,0.006805333619316419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,511,0.006778666749596596
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,15,0.006709333509206772
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,1023,0.00890666681031386
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,1023,0.00877333308259646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,2047,0.00873066671192646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,2047,0.010581333190202713
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,4095,0.010879999647537867
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,4095,0.012725333372751871
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,8191,0.013056000073750814
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,8191,0.027274665733178455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,16383,0.02701333413521449
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,16383,0.050330668687820435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,2047,0.008853333070874214
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,1,0.014757333944241205
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,1,0.01268799975514412
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,3,0.014901333798964819
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,3,0.013066666821638743
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,8191,0.013023999830087027
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,7,0.012666666259368261
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,15,0.01488000030318896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,15,0.0129120002190272
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,31,0.014906667172908783
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,31,0.012890666723251343
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,63,0.015125333021084467
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,63,0.01293333371480306
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,127,0.014741333822409311
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,127,0.012778667112191519
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,255,0.015034666905800501
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,255,0.012736000120639801
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,511,0.016837333639462788
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,511,0.027552001178264618
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,1,0.017018667111794155
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,1,0.015103999525308609
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,1023,0.04665066798528036
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,3,0.017125333348910015
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,7,0.014943999548753103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,15,0.017162666966517765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,3,0.015178666760524115
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,7,0.017231999586025875
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,31,0.016821333517630894
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,7,0.014757333944241205
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,31,0.01492799942692121
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,15,0.015013333410024643
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,63,0.016938666502634685
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,63,0.015050667027632395
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,127,0.014912000546852747
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,127,0.01716800034046173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,255,0.028912000358104706
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,255,0.014874666929244995
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,1023,0.046762665112813316
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,1023,0.08201600114504497
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,1023,0.029370665550231934
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,511,0.0510453333457311
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,511,0.0276053324341774
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,1,0.021914665897687275
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,1,0.017386666188637417
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,3,0.021136000752449036
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,3,0.01728533332546552
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,15,0.016927999754746754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,31,0.02219199885924657
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,31,0.016885332763195038
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,15,0.02288000037272771
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,7,0.017690667261679966
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,63,0.02124800036350886
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,63,0.016805333395799
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,127,0.01695999999841054
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,127,0.02218666672706604
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,1,0.02739733209212621
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,1,0.022991999983787537
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,255,0.027642667293548584
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,255,0.017237332959969837
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,3,0.02717866748571396
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,3,0.023056000471115112
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,7,0.027162666122118633
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,7,0.023221333821614582
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,15,0.027429332335789997
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,31,0.023130667706330616
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,7,0.02306666721900304
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,31,0.02737066646416982
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,63,0.02758399893840154
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,63,0.023242667317390442
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,127,0.02306666721900304
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,127,0.03995733211437861
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,255,0.056373332937558494
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,255,0.02831999957561493
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,1,0.037589333951473236
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,1,0.027258666853109997
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,3,0.03733866661787033
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,3,0.02701333413521449
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,7,0.037434667348861694
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,15,0.023168000082174938
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,15,0.027514666318893433
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,31,0.03774400055408478
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,7,0.027093333502610523
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,31,0.027269333600997925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,63,0.03731200098991394
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,63,0.027093333502610523
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,127,0.038922667503356934
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,1,0.039488000174363456
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,1,0.049178664882977806
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,127,0.02738133321205775
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,3,0.04894933104515076
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,3,0.039749334255854286
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,15,0.03763733307520548
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,7,0.04977599779764811
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,7,0.039706667264302574
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,15,0.03938666731119156
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,15,0.048911998669306435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,31,0.039434666434923805
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,31,0.04972266654173533
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,1,0.007109333450595538
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,1,0.006768000001708667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,3,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,3,0.007050666958093643
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,7,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,63,0.050106664498647056
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,7,0.00725333330531915
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,15,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,15,0.007018666714429855
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,31,0.00679466687142849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,31,0.006831999868154526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,63,0.039461334546407066
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,63,0.0069866664707660675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,127,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,127,0.007055999711155891
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,255,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,255,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,511,0.00877333308259646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,1023,0.00972800018886725
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,511,0.00874133345981439
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,2047,0.010869332899649939
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,2047,0.009216000015536943
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,4095,0.012821332861979803
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,4095,0.010741333166758219
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,8191,0.027024000883102417
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,8191,0.014912000546852747
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,16383,0.04555733501911163
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,127,0.05657599866390228
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,16383,0.028079998989899952
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,1,0.006768000001708667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,1,0.007135999699433644
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,127,0.040789333482583366
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,3,0.007104000076651573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,3,0.0069386667261521024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,7,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,15,0.00660800002515316
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,15,0.006704000135262807
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,31,0.007130666946371396
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,31,0.006655999769767125
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,7,0.006746666505932808
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,63,0.007082666580875714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,63,0.007007999966541926
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,127,0.00684799998998642
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,127,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,255,0.006735999758044879
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,255,0.007007999966541926
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,511,0.008885333314538002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,511,0.00785600021481514
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,1023,0.010170666500926018
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,1023,0.009039999917149544
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,2047,0.011253333340088526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,2047,0.010714666297038397
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,4095,0.02697066714366277
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,4095,0.012901333471139273
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,8191,0.05053866902987162
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,8191,0.027290667096773785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,16383,0.04553600152333578
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,16383,0.08064533273379008
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,63,0.0069919998447100324
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,1023,0.009077333534757296
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,float16,1,0.06830400228500366
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,1,0.047210668524106346
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,3,0.04683200021584829
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,float16,3,0.06818133095900218
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,float16,7,0.06824000179767609
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,7,0.04580800235271454
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,float16,15,0.0682773341735204
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,15,0.04645866652329763
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,float16,31,0.0684746652841568
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,31,0.04753066599369049
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,1,0.07366933425267537
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,float16,1,0.08931199709574382
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,float16,63,0.06809600194295247
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,63,0.04641066491603851
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,float16,3,0.08912533521652222
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,float16,7,0.08930133779843648
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,7,0.07380266487598419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,3,0.07272000114123027
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,float16,15,0.0897173285484314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,15,0.07259733478228252
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,31,0.07300800085067749
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,float16,63,0.08974400162696838
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,float16,31,0.08868799606959026
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,63,0.07259733478228252
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,float16,1,0.1290666659673055
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,1,0.0867733359336853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,float16,3,0.13005866607030234
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,3,0.08689600229263306
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,float16,7,0.13032000263532004
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,7,0.08715732892354329
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,float16,15,0.13051199913024902
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,15,0.08779199918111165
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,float16,31,0.129120002190272
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,31,0.08595732847849528
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,float16,3,0.16848532358805338
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,3,0.13795733451843262
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,1,0.13802666465441385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,float16,1,0.16671999295552573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,float16,7,0.16765334208806357
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,1,0.008698666468262672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,1,0.006831999868154526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,3,0.008778666456540426
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,7,0.0075573331365982694
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,3,0.007040000210205714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,7,0.007157333195209503
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,7,0.13803733388582864
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,15,0.007338666667540868
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,31,0.008816000074148178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,63,0.007381333038210869
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,63,0.007055999711155891
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,31,0.008218666538596153
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,127,0.007007999966541926
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,127,0.0069386667261521024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,255,0.007002666592597961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,255,0.007082666580875714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,511,0.008976000050703684
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,511,0.008581333483258883
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,1023,0.010816000401973724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,1023,0.009050666665037474
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,2047,0.011226666470368704
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,4095,0.0273333340883255
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,4095,0.016106666376193363
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,8191,0.045642669002215065
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,8191,0.028143999477227528
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,15,0.13783466815948486
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,1,0.007813333223263422
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,1,0.006698666761318843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,float16,15,0.16802134116490683
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,3,0.007167999943097432
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,3,0.007034666836261749
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,15,0.007055999711155891
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,7,0.008639999975760778
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,7,0.00696000022192796
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,15,0.007882666463653246
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,15,0.006735999758044879
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,31,0.007983999947706858
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,31,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,63,0.008618666479984919
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,63,0.006720000257094701
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,127,0.007530666887760162
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,255,0.00877333308259646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,255,0.006784000123540561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,511,0.00895999992887179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,511,0.008682666967312494
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,2047,0.012784000486135483
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,1023,0.010821333775917688
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,1023,0.008938666433095932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,2047,0.012960000584522883
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,2047,0.025418666501839954
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,4095,0.05566933254400889
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,4095,0.026341333985328674
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,float16,31,0.168287992477417
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,8191,0.04484266539414724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,8191,0.08041599889596303
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,1,0.00892800030608972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,1,0.007071999832987785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,3,0.0074506668994824094
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,7,0.00697066696981589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,15,0.008762666955590248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,3,0.009008000294367472
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,15,0.007151999821265538
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,31,0.008736000085870424
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,31,0.008143999924262365
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,31,0.1378506620724996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,63,0.00890666681031386
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,127,0.008703999842206636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,127,0.008629333227872849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,255,0.00891733355820179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,255,0.007354666789372762
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,511,0.010330666477481524
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,127,0.00867733359336853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,1023,0.011109333485364914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,1023,0.010693332801262537
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,2047,0.02714666724205017
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,511,0.00884799969693025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,1,0.008869333192706108
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,4095,0.044922664761543274
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,4095,0.027136000494162243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,3,0.009242666885256767
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,3,0.008506666868925095
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,7,0.008805333326260248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,7,0.008890666688481966
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,15,0.009237333511312803
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,15,0.008778666456540426
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,31,0.008762666955590248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,31,0.008698666468262672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,63,0.00898133342464765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,7,0.008821333448092142
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,63,0.008559999987483025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,127,0.008858666444818178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,127,0.009162666896979014
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,255,0.008853333070874214
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,255,0.008778666456540426
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,511,0.011359999577204386
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,63,0.007247999931375186
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,511,0.010885333021481832
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,1023,0.026661333938439686
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,1023,0.01314666618903478
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,2047,0.04354666670163473
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,1,0.006709333509206772
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,1,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,2047,0.026943999032179516
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,3,0.0068693334857622785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,4095,0.08078933258851369
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,3,0.008143999924262365
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,7,0.007093333328763644
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,4095,0.0455626646677653
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,7,0.006831999868154526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,15,0.0068693334857622785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,15,0.006682666639486949
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,31,0.006821333120266597
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,31,0.007397333160042763
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,63,0.006762666627764702
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,63,0.006746666505932808
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,2047,0.012928000340859095
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,127,0.00696000022192796
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,127,0.006613333399097125
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,255,0.00684799998998642
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,255,0.007082666580875714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,511,0.008623999853928884
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,511,0.008586666857202848
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,1023,0.008762666955590248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,1023,0.008639999975760778
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,2047,0.008698666468262672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,2047,0.008890666688481966
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,4095,0.009072000160813332
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,4095,0.008618666479984919
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,1,0.008863999818762144
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,8191,0.009056000038981438
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,16383,0.014837333311637243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,16383,0.01481066644191742
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,1,0.006693333387374878
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,1,0.006837333242098491
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,3,0.006821333120266597
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,7,0.006682666639486949
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,7,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,15,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,15,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,31,0.006602666651209195
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,31,0.006565333033601443
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,63,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,63,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,127,0.006810666372378667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,127,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,255,0.006714666883150737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,511,0.008639999975760778
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,511,0.007226666435599327
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,1023,0.008725333337982496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,1023,0.008714666590094566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,2047,0.008597333605090777
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,4095,0.008634666601816813
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,4095,0.008832000195980072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,8191,0.012741333494583765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,8191,0.010853332777818045
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,16383,0.014858666807413101
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,16383,0.013237333546082178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,1,0.006773333375652631
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,1,0.006725333631038666
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,3,0.00706666645904382
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,3,0.006858666737874349
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,7,0.007055999711155891
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,7,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,15,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,15,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,31,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,63,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,63,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,127,0.00697066696981589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,127,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,8191,0.010757333288590113
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,255,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,3,0.006735999758044879
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,511,0.008687999720374743
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,511,0.008634666601816813
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,1023,0.008922666932145754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,1023,0.008805333326260248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,2047,0.008992000172535578
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,2047,0.008949333180983862
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,4095,0.00890666681031386
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,4095,0.00884799969693025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,255,0.006650666395823161
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,8191,0.012991999586423239
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,8191,0.012752000242471695
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,16383,0.014874666929244995
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,1,0.00697066696981589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,2047,0.008762666955590248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,1,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,3,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,7,0.006762666627764702
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,7,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,3,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,15,0.0069919998447100324
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,15,0.006714666883150737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,31,0.006789333497484525
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,63,0.006821333120266597
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,127,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,127,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,255,0.006751999879876773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,255,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,511,0.008858666444818178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,511,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,31,0.006704000135262807
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,1023,0.008736000085870424
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,1023,0.008842666943868002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,2047,0.00871999996403853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,4095,0.010778666784365972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,4095,0.010762666662534079
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,255,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,8191,0.013679999858140945
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,8191,0.013007999708255133
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,16383,0.0391893337170283
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,1,0.010687999427318573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,1,0.009072000160813332
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,3,0.010965333630641302
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,3,0.008703999842206636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,16383,0.015253332753976187
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,7,0.010805333654085795
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,7,0.00933333362142245
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,15,0.010608000059922537
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,15,0.009029333169261614
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,31,0.010672000547250112
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,31,0.008986666798591614
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,63,0.010821333775917688
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,63,0.008901333436369896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,31,0.006837333242098491
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,127,0.010885333021481832
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,63,0.007029333462317784
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,127,0.009098666409651438
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,255,0.010634666929642359
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,255,0.00892800030608972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,511,0.011130666981140772
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,1023,0.027690666417280834
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,1023,0.014981333166360855
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,2047,0.02828799933195114
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,1,0.012858666479587555
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,16383,0.014789332946141561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,3,0.011765333513418833
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,1,0.010938666760921478
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,3,0.012154666086037954
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,7,0.011231999844312668
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,7,0.012815999488035837
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,15,0.012991999586423239
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,15,0.010842667271693548
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,31,0.010773333410422007
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,63,0.0107893335322539
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,63,0.012736000120639801
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,127,0.012815999488035837
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,127,0.0107893335322539
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,255,0.013072000195582708
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,255,0.010751999914646149
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,511,0.013050666699806849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,511,0.02720000098148982
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,2047,0.045423999428749084
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,511,0.013951999445756277
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,2047,0.008912000184257826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,1,0.00684799998998642
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,3,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,3,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,2047,0.08260799944400787
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,7,0.006837333242098491
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,1023,0.045514668027559914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,1023,0.027087998886903126
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,7,0.006757333253820737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,1,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,15,0.006810666372378667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,31,0.006831999868154526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,31,0.007034666836261749
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,63,0.007226666435599327
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,127,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,127,0.007007999966541926
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,255,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,255,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,511,0.008874666566650072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,511,0.008805333326260248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,1023,0.008736000085870424
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,1023,0.008890666688481966
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,2047,0.008821333448092142
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,2047,0.008858666444818178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,4095,0.012058666596810022
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,31,0.012736000120639801
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,4095,0.01091733326514562
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,8191,0.015130666395028433
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,8191,0.013077333569526672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,16383,0.02979733298222224
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,16383,0.017125333348910015
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,1,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,1,0.00684799998998642
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,3,0.007658666620651881
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,3,0.0069386667261521024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,7,0.0069919998447100324
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,7,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,15,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,15,0.006837333242098491
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,31,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,31,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,63,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,63,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,127,0.007114666824539502
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,127,0.006666666517655055
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,255,0.00701333334048589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,255,0.006741333131988843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,511,0.008687999720374743
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,511,0.00707733320693175
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,1023,0.00901333304742972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,1023,0.008746666833758354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,2047,0.010874666273593903
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,2047,0.008912000184257826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,4095,0.012815999488035837
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,8191,0.035904000202814736
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,8191,0.013034666577974955
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,16383,0.027295999228954315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,16383,0.04930666585763296
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,15,0.007050666958093643
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,1,0.01482133318980535
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,1,0.012383999923865
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,3,0.015087999403476715
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,3,0.012752000242471695
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,7,0.014874666929244995
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,7,0.012837332983811697
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,15,0.014159999787807465
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,15,0.01301866645614306
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,31,0.012357333054145178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,63,0.014896000425020853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,63,0.01313599944114685
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,127,0.01498666654030482
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,127,0.012421333541472753
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,255,0.01488000030318896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,255,0.012389333297808966
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,4095,0.010911999891201654
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,511,0.02758399893840154
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,511,0.015013333410024643
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,63,0.007354666789372762
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,1,0.017050666113694508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,1023,0.04612799982229868
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,1023,0.029146666328112285
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,1,0.015077333897352219
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,3,0.01699200024207433
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,3,0.014736000448465347
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,31,0.015018666783968607
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,15,0.017210666090250015
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,15,0.014815999815861383
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,31,0.016810666769742966
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,7,0.01699200024207433
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,7,0.015087999403476715
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,31,0.015029333531856537
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,63,0.01722666621208191
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,63,0.014874666929244995
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,127,0.017279999951521557
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,127,0.014943999548753103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,255,0.015066667149464289
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,255,0.03376533339420954
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,2047,0.045519997676213585
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,511,0.045706664522488914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,511,0.027386667827765148
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,1023,0.08077866832415263
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,1023,0.046762665112813316
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,1,0.022570667167504627
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,1,0.017210666090250015
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,3,0.021354667842388153
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,3,0.016917333006858826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,15,0.021594665944576263
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,15,0.0170666662355264
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,31,0.022181332111358643
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,31,0.016949333250522614
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,7,0.02327466756105423
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,63,0.02107733239730199
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,63,0.017093333105246227
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,127,0.01692266638080279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,255,0.02736533433198929
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,1,0.027349332968393963
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,1,0.02314666658639908
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,3,0.027269333600997925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,3,0.02309333284695943
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,7,0.02319466571013133
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,15,0.027087998886903126
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,7,0.027349332968393963
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,15,0.02327466756105423
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,31,0.027232001225153606
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,7,0.01709866647919019
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,31,0.023183998962243397
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,63,0.027434666951497395
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,127,0.022053333620230358
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,63,0.023354666928450268
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,127,0.034527999659379326
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,255,0.017082666357358296
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,255,0.05713599920272827
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,255,0.027424000203609467
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,1,0.027493332823117573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,1,0.03757333258787791
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,3,0.03826666623353958
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,3,0.0272533322374026
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,7,0.03755199909210205
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,7,0.027424000203609467
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,15,0.037621334195137024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,127,0.02295999974012375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,15,0.027424000203609467
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,31,0.03743999948104223
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,63,0.037317333122094475
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,63,0.02739733209212621
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,127,0.037621334195137024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,1,0.03937066594759623
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,1,0.049829334020614624
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,3,0.048858667413393654
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,3,0.039706667264302574
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,7,0.04990399877230326
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,31,0.027386667827765148
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,15,0.04809600114822388
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,7,0.03974399964014689
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,15,0.03938133269548416
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,31,0.04919999837875366
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,1,0.0069973332186539965
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,31,0.03955200066169103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,1,0.006981333096822103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,3,0.006725333631038666
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,7,0.006784000123540561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,3,0.00702400008837382
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,127,0.02733866622050603
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,15,0.007002666592597961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,15,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,63,0.04921066761016846
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,31,0.0069973332186539965
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,31,0.007040000210205714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,63,0.00684799998998642
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,127,0.006746666505932808
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,127,0.00696000022192796
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,63,0.0069226666043202085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,63,0.039605334401130676
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,255,0.00696000022192796
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,255,0.006698666761318843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,511,0.008869333192706108
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,511,0.008954666554927826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,1023,0.01028266673286756
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,1023,0.00892800030608972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,2047,0.009258666386206945
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,127,0.05271466573079427
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,4095,0.012917333592971167
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,127,0.04152533411979675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,8191,0.027429332335789997
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,8191,0.015066667149464289
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,1,0.00679466687142849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,1,0.006837333242098491
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,3,0.006837333242098491
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,16383,0.04558933277924856
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,16383,0.02775466690460841
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,7,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,7,0.007221333061655362
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,15,0.007018666714429855
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,15,0.00660800002515316
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,7,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,31,0.006720000257094701
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,31,0.007061333085099856
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,63,0.006682666639486949
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,127,0.0068693334857622785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,127,0.007818666597207388
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,255,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,255,0.007055999711155891
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,511,0.008805333326260248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,511,0.007018666714429855
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,1023,0.009413333609700203
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,1023,0.009045333291093508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,2047,0.010725333044926325
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,2047,0.011141333729028702
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,4095,0.028778667251269024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,8191,0.055434669057528176
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,8191,0.02722666660944621
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,16383,0.0848479966322581
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,16383,0.045498669147491455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,3,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,63,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,2047,0.010832000523805618
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,4095,0.01310933381319046
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,4095,0.01102399950226148
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,1,0.046442667643229164
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,float16,1,0.06834133466084798
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,float16,3,0.0683786670366923
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,3,0.04573333263397217
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,float16,7,0.06825066606203715
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,7,0.04651199777921041
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,float16,15,0.06844266752401988
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,15,0.047210668524106346
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,float16,31,0.068271999557813
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,31,0.04580800235271454
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,float16,63,0.06807466844717662
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,63,0.046021332343419395
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,float16,1,0.08871466914812724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,1,0.07287466526031494
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,float16,3,0.08896533648173015
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,3,0.07344000041484833
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,7,0.07434133191903432
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,float16,15,0.08920533458391826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,15,0.07374933362007141
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,float16,31,0.08985599875450134
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,31,0.07246399919191997
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,float16,7,0.0890880028406779
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,float16,63,0.08918933073679607
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,63,0.07228800157705943
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,1,0.08667733271916707
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,float16,1,0.13089600205421448
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,3,0.08674666285514832
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,float16,3,0.130431999762853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,7,0.08662399649620056
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,float16,7,0.13110933701197305
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,float16,15,0.12980799873669943
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,15,0.08777599533398946
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,float16,31,0.12965866923332214
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,31,0.08664533495903015
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,float16,1,0.1679733395576477
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,1,0.13795199990272522
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,float16,3,0.16821332772572836
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,3,0.13801599542299905
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,1,0.008693333094318708
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,1,0.007002666592597961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,3,0.008597333605090777
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,3,0.007045333584149678
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,7,0.008623999853928884
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,7,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,15,0.00860799973209699
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,float16,7,0.16804800430933634
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,15,0.007040000210205714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,31,0.008485333373149237
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,31,0.00696000022192796
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,63,0.008634666601816813
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,7,0.13792533675829569
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,127,0.007957333077987036
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,63,0.007040000210205714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,127,0.007104000076651573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,255,0.007407999907930692
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,255,0.006768000001708667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,511,0.008938666433095932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,511,0.00877333308259646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,1023,0.010640000303586325
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,1023,0.00878399983048439
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,2047,0.012703999876976013
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,2047,0.010933333386977514
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,4095,0.02736533433198929
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,4095,0.01471466695268949
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,8191,0.045791998505592346
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,8191,0.0281333327293396
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,1,0.00766933336853981
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,1,0.007029333462317784
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,3,0.00878399983048439
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,3,0.006650666395823161
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,7,0.008805333326260248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,7,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,15,0.007242666557431221
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,15,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,31,0.008623999853928884
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,31,0.0069226666043202085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,63,0.008378666515151659
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,63,0.00697066696981589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,127,0.008325333396593729
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,float16,15,0.16887466112772623
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,127,0.0068853336075941724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,255,0.008687999720374743
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,255,0.006671999891599019
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,15,0.1383093297481537
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,511,0.008954666554927826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,1023,0.009119999905427298
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,float16,31,0.16796799500783285
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,1023,0.011098666737476984
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,511,0.008629333227872849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,2047,0.012906666845083237
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,4095,0.026288000245889027
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,2047,0.025637333591779072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,8191,0.04445866743723551
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,8191,0.08040533463160197
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,1,0.007093333328763644
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,3,0.007311999797821045
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,31,0.1378773351510366
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,3,0.006714666883150737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,7,0.007391999786098798
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,1,0.00679466687142849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,7,0.006720000257094701
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,15,0.007247999931375186
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,15,0.007002666592597961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,31,0.0069226666043202085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,31,0.007018666714429855
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,63,0.00701333334048589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,63,0.00701333334048589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,127,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,255,0.008725333337982496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,511,0.008714666590094566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,511,0.008767999708652496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,1023,0.010944000134865442
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,1023,0.008837333569924036
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,2047,0.012800000607967377
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,127,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,4095,0.02738133321205775
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,2047,0.010826667149861654
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,4095,0.014933332800865173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,8191,0.04572799801826477
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,1,0.008597333605090777
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,3,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,8191,0.02815466622511546
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,7,0.008602666358153025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,7,0.00679466687142849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,15,0.008570666735370954
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,15,0.007050666958093643
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,31,0.00808533343176047
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,31,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,63,0.008709333216150602
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,63,0.008021333565314611
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,127,0.008837333569924036
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,127,0.006773333375652631
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,255,0.008725333337982496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,255,0.006704000135262807
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,511,0.009008000294367472
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,511,0.00895999992887179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,1023,0.01099733387430509
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,1023,0.008693333094318708
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,2047,0.026816000541051228
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,2047,0.012949333836634954
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,8191,0.08045866588751475
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,4095,0.026975999275843304
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,255,0.006805333619316419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,8191,0.045754666129748024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,1,0.00916800027092298
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,3,0.008832000195980072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,3,0.008869333192706108
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,7,0.008901333436369896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,7,0.008879999940594038
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,1,0.00684799998998642
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,15,0.009045333291093508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,3,0.008757333581646284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,15,0.008901333436369896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,31,0.00895999992887179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,31,0.008623999853928884
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,63,0.008805333326260248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,63,0.00871999996403853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,127,0.008992000172535578
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,127,0.00897066667675972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,255,0.008639999975760778
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,4095,0.04372799893220266
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,511,0.011717333147923151
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,511,0.010890666395425797
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,1023,0.01293333371480306
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,2047,0.04412800073623657
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,2047,0.027189334233601887
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,4095,0.08582400282224019
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,4095,0.04659200211366018
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,4095,0.0439626673857371
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,1,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,8191,0.15424000223477682
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,1,0.0069226666043202085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,8191,0.08206399778525035
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,3,0.006688000013430913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,3,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,1,0.008816000074148178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,15,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,15,0.00697066696981589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,31,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,63,0.006858666737874349
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,63,0.006645333642760913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,127,0.0068693334857622785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,127,0.006751999879876773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,255,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,255,0.006629333520929019
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,511,0.00890666681031386
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,511,0.007104000076651573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,1023,0.009178666397929192
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,1023,0.008698666468262672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,2047,0.008901333436369896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,255,0.008874666566650072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,2047,0.0086666668454806
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,4095,0.008943999807039896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,1023,0.02651199946800868
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,4095,0.008874666566650072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,8191,0.008949333180983862
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,16383,0.01402666668097178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,16383,0.013072000195582708
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,32767,0.017210666090250015
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,1,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,1,0.007071999832987785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,3,0.007114666824539502
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,3,0.006629333520929019
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,7,0.00666133314371109
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,7,0.006624000146985054
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,15,0.006821333120266597
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,15,0.006858666737874349
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,31,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,31,0.006570666407545407
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,7,0.007050666958093643
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,63,0.006837333242098491
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,63,0.006735999758044879
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,7,0.00696000022192796
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,31,0.007290666922926903
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,127,0.00697066696981589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,255,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,255,0.006720000257094701
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,511,0.008634666601816813
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,511,0.007418666655818622
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,1023,0.008938666433095932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,2047,0.008943999807039896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,2047,0.008538666491707167
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,4095,0.009018666421373686
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,4095,0.008602666358153025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,8191,0.01091733326514562
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,8191,0.008992000172535578
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,16383,0.014389333625634512
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,32767,0.017050666113694508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,32767,0.015061333775520325
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,1,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,1,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,3,0.006981333096822103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,3,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,7,0.006741333131988843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,8191,0.010805333654085795
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,7,0.006533333410819371
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,15,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,15,0.006698666761318843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,32767,0.01878400022784869
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,31,0.006570666407545407
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,31,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,63,0.00679466687142849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,63,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,127,0.007941333577036858
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,127,0.007135999699433644
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,255,0.006773333375652631
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,255,0.006655999769767125
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,511,0.006981333096822103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,511,0.00873066671192646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,1023,0.010255999863147736
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,1023,0.008527999743819237
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,2047,0.008826666822036108
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,2047,0.008656000097592672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,4095,0.008709333216150602
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,4095,0.008869333192706108
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,8191,0.012074666718641916
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,8191,0.011034666250149408
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,16383,0.014906667172908783
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,16383,0.013056000073750814
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,1,0.00707733320693175
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,32767,0.03749866783618927
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,127,0.006757333253820737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,3,0.007296000296870868
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,1,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,3,0.007055999711155891
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,7,0.006730666384100914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,7,0.007045333584149678
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,15,0.0069226666043202085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,1023,0.008725333337982496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,15,0.00702400008837382
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,31,0.00701333334048589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,63,0.006730666384100914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,63,0.006751999879876773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,127,0.00696000022192796
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,127,0.0069973332186539965
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,255,0.007034666836261749
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,255,0.006645333642760913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,16383,0.012970666090647379
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,511,0.008746666833758354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,1023,0.008922666932145754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,1023,0.008549333239595095
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,2047,0.009056000038981438
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,2047,0.008687999720374743
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,4095,0.008896000062425932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,4095,0.008634666601816813
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,8191,0.010687999427318573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,8191,0.009018666421373686
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,16383,0.01498666654030482
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,32767,0.019098666807015736
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,32767,0.016970666746298473
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,1,0.006746666505932808
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,3,0.006704000135262807
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,3,0.006602666651209195
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,7,0.006784000123540561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,7,0.006586666529377301
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,15,0.006698666761318843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,15,0.006645333642760913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,31,0.006746666505932808
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,31,0.006495999793211619
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,63,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,63,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,127,0.006741333131988843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,127,0.006645333642760913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,255,0.0069386667261521024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,32767,0.015765332927306492
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,255,0.006533333410819371
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,511,0.008703999842206636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,511,0.007071999832987785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,1023,0.008746666833758354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,31,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,2047,0.008703999842206636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,2047,0.008570666735370954
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,4095,0.00890666681031386
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,4095,0.008762666955590248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,511,0.008933333059151968
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,8191,0.01089599976936976
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,16383,0.014783999572197596
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,16383,0.013141332815090815
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,32767,0.029264000554879505
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,32767,0.015919999529918034
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,16383,0.014229333649079004
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,1,0.006682666639486949
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,1,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,3,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,3,0.00701333334048589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,1,0.006597333277265231
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,7,0.0068853336075941724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,7,0.006800000245372455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,15,0.006784000123540561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,15,0.006773333375652631
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,31,0.0069866664707660675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,63,0.006789333497484525
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,63,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,127,0.007087999954819679
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,127,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,255,0.00707733320693175
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,511,0.008832000195980072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,511,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,1023,0.00903466654320558
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,2047,0.00877333308259646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,2047,0.008949333180983862
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,4095,0.010901333143313726
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,4095,0.0107893335322539
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,8191,0.013829333086808523
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,8191,0.012944000462690989
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,16383,0.032986665765444435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,1023,0.008853333070874214
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,16383,0.015157333264748255
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,32767,0.05657066901524862
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,32767,0.02920000006755193
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,1,0.00860799973209699
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,1,0.007087999954819679
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,3,0.00891733355820179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,8191,0.012144000579913458
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,3,0.007589333380262057
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,7,0.008581333483258883
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,7,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,15,0.008992000172535578
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,15,0.007647999872763951
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,31,0.00877333308259646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,31,0.008698666468262672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,63,0.008896000062425932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,63,0.00867733359336853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,127,0.008736000085870424
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,127,0.008592000231146812
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,31,0.00679466687142849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,255,0.009050666665037474
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,255,0.008896000062425932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,511,0.008912000184257826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,255,0.006671999891599019
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,511,0.00871999996403853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,1023,0.008858666444818178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,1023,0.011071999867757162
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,4095,0.045466666420300804
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,2047,0.027290667096773785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,4095,0.0276853342851003
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,1,0.00890666681031386
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,1,0.008832000195980072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,3,0.008874666566650072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,7,0.00901333304742972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,7,0.008703999842206636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,3,0.00891733355820179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,15,0.008693333094318708
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,15,0.00892800030608972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,31,0.008863999818762144
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,31,0.00874133345981439
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,63,0.009018666421373686
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,63,0.008613333106040955
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,127,0.008954666554927826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,127,0.008826666822036108
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,255,0.008938666433095932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,255,0.008986666798591614
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,511,0.012165332833925882
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,511,0.010709332923094431
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,1023,0.027087998886903126
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,2047,0.04516266783078512
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,2047,0.0271519993742307
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,4095,0.08117333551247914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,1,0.010885333021481832
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,1023,0.011002667248249054
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,1,0.011050666371981302
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,4095,0.046240001916885376
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,3,0.012069333344697952
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,2047,0.013077333569526672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,3,0.010858666151762009
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,7,0.011050666371981302
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,15,0.012330666184425354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,7,0.012784000486135483
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,15,0.010960000256697336
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,31,0.010784000158309937
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,63,0.012293333808581034
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,63,0.01090666651725769
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,31,0.012858666479587555
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,127,0.010938666760921478
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,127,0.012655999511480331
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,255,0.010768000036478043
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,255,0.012458667159080505
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,1023,0.04572799801826477
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,1023,0.027722666660944622
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,511,0.03581333408753077
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,511,0.013023999830087027
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,2047,0.08165333171685536
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,1,0.006805333619316419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,3,0.006773333375652631
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,2047,0.045509333411852516
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,7,0.007055999711155891
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,1023,0.012682666381200155
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,4095,0.0819413314263026
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,7,0.00696000022192796
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,15,0.006831999868154526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,15,0.007029333462317784
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,63,0.006789333497484525
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,4095,0.15576533476511636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,127,0.007157333195209503
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,3,0.007082666580875714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,1,0.00701333334048589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,31,0.007018666714429855
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,127,0.006858666737874349
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,255,0.006677333265542984
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,255,0.00684799998998642
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,511,0.008837333569924036
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,1023,0.008661333471536636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,2047,0.008693333094318708
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,2047,0.00897066667675972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,4095,0.008885333314538002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,4095,0.008725333337982496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,8191,0.012981332838535309
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,8191,0.012725333372751871
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,16383,0.016010666886965435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,16383,0.014874666929244995
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,32767,0.030794667700926464
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,32767,0.01886933296918869
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,1,0.007173333317041397
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,1,0.006858666737874349
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,3,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,7,0.006768000001708667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,7,0.006671999891599019
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,15,0.007082666580875714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,15,0.0069866664707660675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,31,0.00684799998998642
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,31,0.006709333509206772
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,63,0.00696000022192796
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,63,0.006704000135262807
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,127,0.006741333131988843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,127,0.006688000013430913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,255,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,255,0.006688000013430913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,511,0.00895999992887179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,511,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,1023,0.00877333308259646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,1023,0.00890666681031386
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,2047,0.008858666444818178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,2047,0.008682666967312494
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,4095,0.010970667004585266
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,63,0.0069866664707660675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,4095,0.010618666807810465
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,8191,0.015114666273196539
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,8191,0.012944000462690989
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,511,0.008709333216150602
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,16383,0.0284853329261144
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,16383,0.01509333277742068
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,32767,0.04684266448020935
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,32767,0.029466666281223297
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,1,0.007082666580875714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,1,0.006677333265542984
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,3,0.006789333497484525
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,3,0.006778666749596596
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,7,0.0069866664707660675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,3,0.0069386667261521024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,15,0.006677333265542984
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,15,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,31,0.007093333328763644
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,31,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,63,0.006784000123540561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,63,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,127,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,127,0.007050666958093643
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,255,0.007002666592597961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,255,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,511,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,1023,0.008853333070874214
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,1023,0.008746666833758354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,2047,0.010949333508809408
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,2047,0.008810666700204214
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,4095,0.010842667271693548
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,8191,0.03670933345953623
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,8191,0.012901333471139273
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,16383,0.05533866584300995
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,16383,0.027215999861558277
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,31,0.00697066696981589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,32767,0.08157866696516673
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,32767,0.045184001326560974
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,1023,0.008576000109314919
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,1,0.011168000598748526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,1,0.008976000050703684
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,3,0.010933333386977514
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,3,0.008954666554927826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,7,0.010656000425418219
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,7,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,7,0.008949333180983862
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,15,0.008922666932145754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,31,0.010597333312034607
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,31,0.008767999708652496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,63,0.010832000523805618
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,63,0.009141333401203156
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,127,0.01081066702802976
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,127,0.009136000027259191
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,511,0.008879999940594038
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,255,0.010645333677530289
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,255,0.00902399979531765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,4095,0.012794667234023413
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,511,0.01293333371480306
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,511,0.011333333949247995
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,1023,0.027077332139015198
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,1023,0.015114666273196539
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,1,0.012890666723251343
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,1,0.010885333021481832
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,2047,0.04577599962552389
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,7,0.01089599976936976
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,3,0.010773333410422007
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,7,0.012400000045696894
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,15,0.012773333738247553
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,15,0.01071999967098236
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,31,0.011893333246310553
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,31,0.010874666273593903
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,15,0.010922666639089584
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,63,0.012965332716703415
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,63,0.010992000500361124
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,127,0.012389333297808966
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,127,0.010693332801262537
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,255,0.012671999633312225
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,255,0.010757333288590113
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,511,0.02735466758410136
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,511,0.014581333845853806
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,2047,0.0823359986146291
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,1023,0.04554133117198944
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,2047,0.04607999821503957
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,3,0.0129120002190272
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,1,0.017103999853134155
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,3,0.01682666689157486
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,7,0.016693333784739178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,7,0.015040000279744467
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,3,0.015029333531856537
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,15,0.017077332983414333
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,15,0.01462399959564209
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,31,0.014938666174809137
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,31,0.017093333105246227
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,63,0.017221332838137943
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,63,0.014746667196353277
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,127,0.016778666526079178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,127,0.014906667172908783
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,255,0.02935466667016347
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,255,0.014730667074521383
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,1023,0.027482666075229645
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,2047,0.029450667401154835
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,1,0.01595199977358182
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,511,0.027232001225153606
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,511,0.04452266792456309
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,1023,0.04548266530036926
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,1023,0.08100266754627228
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,1,0.01309866706530253
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,2047,0.15318399667739868
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,1,0.014826666563749313
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,7,0.014405333747466406
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,7,0.01303999995191892
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,15,0.015189333508412043
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,15,0.0129120002190272
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,3,0.014912000546852747
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,3,0.012863999853531519
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,31,0.014805333067973455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,31,0.012608000387748083
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,63,0.014858666807413101
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,63,0.01303999995191892
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,127,0.014794666320085526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,127,0.012757333616415659
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,255,0.013221333424250284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,2047,0.0825439989566803
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,511,0.027424000203609467
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,511,0.016986666868130367
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,1,0.01709866647919019
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,3,0.01711999997496605
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,3,0.014869333555301031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,7,0.017082666357358296
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,7,0.014762666076421738
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,15,0.015114666273196539
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,15,0.017045332739750545
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,31,0.014842666685581207
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,31,0.017130666722853977
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,63,0.016879999389251072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,63,0.014826666563749313
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,255,0.014479999740918478
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,127,0.017162666966517765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,127,0.015050667027632395
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,255,0.027077332139015198
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,255,0.015066667149464289
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,1,0.014853333433469137
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,511,0.04539200166861216
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,511,0.02738133321205775
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,1,0.027466667195161183
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,1,0.023237332701683044
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,3,0.02736533433198929
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,7,0.023152001202106476
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,3,0.023024000227451324
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,7,0.027285332481066387
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,31,0.027215999861558277
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,31,0.023226665953795116
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,63,0.027263998985290527
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,63,0.022976001103719074
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,127,0.02292799949645996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,15,0.027114666998386383
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,127,0.029130667448043823
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,15,0.02309333284695943
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,255,0.05232533315817515
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,255,0.02758399893840154
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,1,0.017103999853134155
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,511,0.08150933186213176
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,511,0.045754666129748024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,1,0.02239466706911723
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,3,0.01716800034046173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,3,0.022976001103719074
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,7,0.023120000958442688
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,7,0.01720533271630605
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,15,0.02109333376089732
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,63,0.021189334491888683
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,31,0.017157333592573803
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,63,0.017194667210181553
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,31,0.021322667598724365
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,15,0.01727466657757759
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,127,0.022133332987626392
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,127,0.017301333447297413
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,1,0.02317333221435547
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,1,0.02736533433198929
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,255,0.027786667148272198
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,255,0.016885332763195038
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,3,0.027424000203609467
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,3,0.0232640008131663
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,7,0.02752000093460083
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,15,0.027093333502610523
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,15,0.023183998962243397
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,7,0.02349333216746648
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,31,0.02714666724205017
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,31,0.02332266668478648
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,63,0.02714666724205017
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,63,0.023120000958442688
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,127,0.029487999776999157
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,127,0.023002666731675465
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,1,0.04776533444722494
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,255,0.04585599899291992
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,255,0.029120000700155895
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,1,0.039503999054431915
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,3,0.049839998284975685
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,3,0.03978666663169861
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,7,0.03933866570393244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,7,0.04957333207130432
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,31,0.0489279975493749
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,15,0.047877331574757896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,15,0.039306665460268654
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,31,0.03923733284076055
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,63,0.04897066454092661
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,1,0.00679466687142849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,3,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,63,0.039264000952243805
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,7,0.007167999943097432
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,3,0.00706666645904382
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,127,0.0510453333457311
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,15,0.006821333120266597
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,31,0.007269333427151044
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,7,0.0069973332186539965
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,127,0.03998400022586187
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,63,0.00679466687142849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,63,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,127,0.007050666958093643
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,31,0.007071999832987785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,255,0.0069919998447100324
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,255,0.0069866664707660675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,511,0.008570666735370954
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,511,0.008709333216150602
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,127,0.007061333085099856
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,1023,0.009050666665037474
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,1023,0.008767999708652496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,2047,0.008879999940594038
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,2047,0.00897066667675972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,4095,0.012554666648308435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,4095,0.01097600037852923
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,8191,0.015109332899252573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,8191,0.012826666235923767
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,16383,0.029717333614826202
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,16383,0.016885332763195038
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,255,0.0491946687301
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,32767,0.047594666481018066
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,255,0.09037866195042928
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,1,0.007007999966541926
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,1,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,3,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,7,0.0069973332186539965
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,15,0.006650666395823161
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,15,0.006698666761318843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,31,0.0069866664707660675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,31,0.006688000013430913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,63,0.006666666517655055
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,63,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,15,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,127,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,255,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,255,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,511,0.008890666688481966
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,511,0.007322666545708974
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,3,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,1023,0.008933333059151968
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,7,0.006655999769767125
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,1023,0.008943999807039896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,2047,0.010762666662534079
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,2047,0.008661333471536636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,4095,0.012981332838535309
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,4095,0.010922666639089584
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,8191,0.02701333413521449
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,8191,0.013023999830087027
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,16383,0.04554666578769684
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,16383,0.02758399893840154
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,32767,0.08152533570925395
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,32767,0.0455626646677653
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,1,0.00701333334048589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,1,0.006591999903321266
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,3,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,3,0.0068693334857622785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,7,0.007002666592597961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,7,0.006634666894872983
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,15,0.006762666627764702
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,15,0.0069866664707660675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,31,0.006981333096822103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,31,0.006762666627764702
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,63,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,63,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,127,0.00697066696981589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,127,0.0069973332186539965
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,1,0.006666666517655055
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,255,0.007162666569153468
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,511,0.008853333070874214
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,511,0.008010666817426682
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,1023,0.00871999996403853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,1023,0.01022933361430963
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,2047,0.01118933285276095
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,2047,0.010981333752473196
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,4095,0.012960000584522883
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,4095,0.026965332527955372
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,127,0.007567999884486198
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,8191,0.027242665489514668
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,8191,0.05068266888459524
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,16383,0.0454773356517156
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,16383,0.08046933511892955
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,32767,0.08158933122952779
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,32767,0.1525226632754008
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,32767,0.030720000465710957
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,255,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,1,0.03721066564321518
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,1,0.02736533433198929
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,3,0.03746666759252548
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,3,0.027402666707833607
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,7,0.037632000943024956
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,7,0.02739199995994568
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,15,0.03734400123357773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,15,0.027034667630990345
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,31,0.027274665733178455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,31,0.0373333344856898
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,63,0.03749333322048187
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,63,0.027434666951497395
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,127,0.03794133414824804
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,1,0.047882666190465294
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,127,0.02741333345572154
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,1,0.03937066594759623
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,3,0.04775999983151754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,3,0.03949866692225138
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,7,0.04981866478919983
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,7,0.039647998909155525
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,15,0.04961599906285604
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,15,0.03944533318281174
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,31,0.04958933095137278
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,31,0.039647998909155525
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,63,0.04850666721661886
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,63,0.03932266682386398
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,127,0.051818668842315674
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,127,0.0415786678592364
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,1,0.08864532907803853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,1,0.07233066856861115
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,3,0.08910399675369263
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,3,0.07241599758466084
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,7,0.0881866713364919
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,7,0.07231999933719635
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,31,0.07229866584142049
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,15,0.08947733044624329
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,31,0.0885759989420573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,63,0.08941866954167683
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,63,0.07213333249092102
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,127,0.09246399998664856
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,15,0.07227733234564464
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,127,0.0722453345855077
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,1,0.04780800143877665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,float16,3,0.06843199829260509
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,3,0.047637333472569786
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,float16,7,0.06966933111349742
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,float16,1,0.06818666557470958
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,7,0.047541335225105286
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,float16,15,0.06824533144632976
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,15,0.048058668772379555
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,float16,31,0.06835733354091644
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,31,0.04774933556715647
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,63,0.04764266808827718
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,float16,1,0.08684266606966655
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,1,0.07216000060240428
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,float16,63,0.06807999809583028
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,float16,3,0.08939733107884724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,3,0.07253333429495494
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,float16,7,0.08875200152397156
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,7,0.07297066847483318
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,15,0.07232533395290375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,float16,15,0.08988266189893086
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,float16,31,0.08852799733479817
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,31,0.07295999924341838
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,float16,1,0.1670560042063395
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,63,0.07262399792671204
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,float16,63,0.08864532907803853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,1,0.13581333557764688
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,float16,3,0.1678719917933146
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,3,0.1360373298327128
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,float16,7,0.16726932922999063
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,1,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,1,0.006666666517655055
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,3,0.007045333584149678
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,3,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,7,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,7,0.006688000013430913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,7,0.13621866703033447
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,15,0.007002666592597961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,15,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,31,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,31,0.006858666737874349
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,63,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,63,0.00684799998998642
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,127,0.007045333584149678
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,127,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,255,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,255,0.00684799998998642
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,511,0.008832000195980072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,511,0.008656000097592672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,15,0.13702932993570963
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,1023,0.008933333059151968
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,float16,15,0.1665013333161672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,1023,0.008821333448092142
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,2047,0.010757333288590113
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,2047,0.008879999940594038
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,float16,31,0.16779200236002603
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,4095,0.012944000462690989
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,4095,0.010965333630641302
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,8191,0.027119999130566914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,16383,0.04598933458328247
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,1,0.0069226666043202085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,16383,0.027477333943049114
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,31,0.13591999808947244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,1,0.006831999868154526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,8191,0.014885333677132925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,3,0.006730666384100914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,3,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,float16,63,0.16692266861597696
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,7,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,7,0.006826666494210561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,15,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,15,0.006682666639486949
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,31,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,63,0.006778666749596596
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,63,0.00697066696981589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,127,0.00697066696981589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,63,0.1360586682955424
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,255,0.006831999868154526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,255,0.006751999879876773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,511,0.008421333506703377
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,511,0.00789866658548514
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,31,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,1023,0.008885333314538002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,2047,0.010863999525705973
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,4095,0.027104000250498455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,4095,0.013093333691358566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,8191,0.045461331804593406
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,8191,0.02720533311367035
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,16383,0.08124800026416779
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,16383,0.04553066690762838
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,1,0.006858666737874349
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,1,0.007925333455204964
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,3,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,3,0.008634666601816813
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,7,0.007754666730761528
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,7,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,15,0.007994666695594788
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,15,0.006821333120266597
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,127,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,31,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,31,0.007760000104705493
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,63,0.006741333131988843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,63,0.008576000109314919
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,127,0.008832000195980072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,127,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,1023,0.00897066667675972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,255,0.007098666702707608
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,255,0.008207999790708223
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,511,0.008896000062425932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,1023,0.009061333412925402
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,1023,0.01097600037852923
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,2047,0.012815999488035837
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,2047,0.025087999800841015
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,4095,0.043696001172065735
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,4095,0.02664000044266383
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,8191,0.04437333345413208
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,8191,0.08196266492207845
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,1,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,16383,0.0803306649128596
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,1,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,16383,0.15218666195869446
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,3,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,7,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,15,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,7,0.007029333462317784
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,2047,0.01099733387430509
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,15,0.006837333242098491
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,31,0.008682666967312494
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,31,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,63,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,127,0.008714666590094566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,255,0.008816000074148178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,511,0.008874666566650072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,511,0.008816000074148178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,1023,0.010650667051474253
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,1023,0.008703999842206636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,2047,0.013007999708255133
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,2047,0.011152000476916632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,4095,0.027258666853109997
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,4095,0.015077333897352219
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,8191,0.04572799801826477
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,8191,0.028378665447235107
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,1,0.00892800030608972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,1,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,511,0.008810666700204214
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,3,0.00761600024998188
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,3,0.00702400008837382
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,7,0.007311999797821045
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,7,0.00696000022192796
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,15,0.008922666932145754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,15,0.006757333253820737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,31,0.008661333471536636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,3,0.007173333317041397
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,31,0.006741333131988843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,63,0.008352000266313553
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,63,0.0069866664707660675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,127,0.00891733355820179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,127,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,255,0.008485333373149237
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,255,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,127,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,511,0.008992000172535578
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,255,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,511,0.008879999940594038
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,1023,0.010853332777818045
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,1023,0.00898133342464765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,2047,0.02621866762638092
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,2047,0.012853333105643591
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,4095,0.04404800136884054
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,4095,0.027242665489514668
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,1,0.008762666955590248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,3,0.008837333569924036
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,1,0.008746666833758354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,8191,0.04553600152333578
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,8191,0.08037333190441132
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,7,0.008816000074148178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,7,0.008997333546479544
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,15,0.009045333291093508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,15,0.008639999975760778
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,31,0.008762666955590248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,31,0.008943999807039896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,63,0.00902399979531765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,63,0.008656000097592672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,127,0.009248000259200731
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,127,0.008602666358153025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,255,0.008885333314538002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,255,0.008874666566650072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,511,0.010944000134865442
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,511,0.010661333799362183
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,1023,0.025642665723959606
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,1023,0.013056000073750814
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,2047,0.0443146675825119
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,2047,0.02703999976317088
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,63,0.00684799998998642
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,4095,0.04573333263397217
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,1,0.007002666592597961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,8191,0.0825973351796468
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,8191,0.15362133582433066
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,4095,0.08056533336639404
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,3,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,3,0.006773333375652631
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,7,0.00684799998998642
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,7,0.006746666505932808
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,15,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,31,0.00696000022192796
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,63,0.007061333085099856
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,63,0.006784000123540561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,127,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,255,0.007157333195209503
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,255,0.006805333619316419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,511,0.008922666932145754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,511,0.008618666479984919
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,1023,0.00890666681031386
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,3,0.009072000160813332
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,2047,0.008853333070874214
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,2047,0.008778666456540426
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,4095,0.008767999708652496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,4095,0.008746666833758354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,8191,0.010992000500361124
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,16383,0.016000000139077503
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,16383,0.01310933381319046
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,32767,0.018112000077962875
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,32767,0.017338667064905167
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,1,0.006784000123540561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,1,0.006773333375652631
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,3,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,3,0.006837333242098491
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,7,0.006768000001708667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,7,0.006821333120266597
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,15,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,31,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,31,0.007391999786098798
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,1,0.00679466687142849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,63,0.006735999758044879
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,63,0.006730666384100914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,15,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,31,0.006709333509206772
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,127,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,127,0.006757333253820737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,127,0.006682666639486949
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,255,0.006858666737874349
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,255,0.006597333277265231
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,511,0.008576000109314919
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,511,0.007007999966541926
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,1023,0.0084906667470932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,1023,0.008912000184257826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,2047,0.008752000207702318
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,1023,0.00891733355820179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,4095,0.009109333157539368
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,4095,0.00890666681031386
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,8191,0.010368000095089277
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,8191,0.009002666920423508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,16383,0.014485333114862442
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,16383,0.01303999995191892
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,8191,0.009077333534757296
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,32767,0.016890666137139004
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,32767,0.01540800059835116
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,1,0.007018666714429855
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,1,0.006826666494210561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,3,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,3,0.006645333642760913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,7,0.006821333120266597
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,7,0.006714666883150737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,15,0.006688000013430913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,15,0.006831999868154526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,31,0.006858666737874349
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,63,0.007050666958093643
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,127,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,127,0.006655999769767125
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,15,0.006773333375652631
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,255,0.00684799998998642
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,511,0.008714666590094566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,511,0.006768000001708667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,1023,0.009002666920423508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,1023,0.008703999842206636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,2047,0.008933333059151968
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,2047,0.00884799969693025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,4095,0.00902399979531765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,4095,0.008992000172535578
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,8191,0.013061333447694778
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,8191,0.010981333752473196
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,16383,0.015040000279744467
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,16383,0.012960000584522883
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,32767,0.039520000418027244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,32767,0.015530666957298914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,1,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,2047,0.008837333569924036
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,1,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,3,0.0069919998447100324
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,3,0.0069386667261521024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,7,0.006831999868154526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,7,0.006586666529377301
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,15,0.006720000257094701
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,31,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,31,0.006634666894872983
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,63,0.007184000064929326
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,127,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,127,0.007109333450595538
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,255,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,255,0.006821333120266597
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,511,0.008762666955590248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,31,0.006645333642760913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,511,0.0074560002734263735
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,1023,0.00898133342464765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,63,0.006602666651209195
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,1023,0.008645333349704742
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,2047,0.008901333436369896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,2047,0.00860799973209699
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,4095,0.009002666920423508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,4095,0.008826666822036108
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,8191,0.010858666151762009
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,8191,0.00891733355820179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,16383,0.014954666296641031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,16383,0.013951999445756277
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,32767,0.018629333625237148
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,32767,0.01721599946419398
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,1,0.006666666517655055
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,3,0.006655999769767125
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,3,0.006634666894872983
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,7,0.006826666494210561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,7,0.006682666639486949
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,15,0.006725333631038666
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,31,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,63,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,63,0.006602666651209195
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,127,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,127,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,255,0.006821333120266597
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,255,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,511,0.009797333429257074
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,511,0.006784000123540561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,1023,0.008842666943868002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,15,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,1023,0.008687999720374743
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,2047,0.008805333326260248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,2047,0.008650666723648706
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,63,0.006810666372378667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,4095,0.008837333569924036
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,4095,0.008853333070874214
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,8191,0.012272000312805176
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,16383,0.01498666654030482
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,32767,0.031231999397277832
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,32767,0.016074666132529575
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,1,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,1,0.006805333619316419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,255,0.007104000076651573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,3,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,3,0.007007999966541926
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,1,0.0068853336075941724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,15,0.006666666517655055
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,7,0.0069386667261521024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,7,0.006698666761318843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,15,0.00684799998998642
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,31,0.007018666714429855
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,15,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,31,0.007114666824539502
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,63,0.007120000198483467
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,63,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,127,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,127,0.006821333120266597
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,255,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,255,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,511,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,511,0.008559999987483025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,1023,0.00874133345981439
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,1023,0.008703999842206636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,2047,0.008693333094318708
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,2047,0.008943999807039896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,4095,0.010858666151762009
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,4095,0.010944000134865442
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,8191,0.014815999815861383
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,8191,0.01268799975514412
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,16383,0.03881066789229711
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,8191,0.011039999624093374
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,32767,0.058362667759259544
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,32767,0.02934933453798294
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,1,0.008576000109314919
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,1,0.008879999940594038
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,3,0.008565333361426989
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,3,0.008826666822036108
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,7,0.008592000231146812
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,7,0.008597333605090777
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,15,0.008879999940594038
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,15,0.00855466661353906
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,31,0.008709333216150602
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,31,0.008874666566650072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,63,0.008634666601816813
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,63,0.008639999975760778
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,127,0.008581333483258883
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,127,0.008901333436369896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,255,0.00867733359336853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,255,0.008576000109314919
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,511,0.008901333436369896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,511,0.008853333070874214
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,1023,0.011349332829316458
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,1023,0.010677333921194077
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,2047,0.02701866626739502
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,2047,0.013077333569526672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,4095,0.04539200166861216
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,4095,0.027087998886903126
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,1,0.008954666554927826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,16383,0.015061333775520325
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,16383,0.01313599944114685
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,7,0.008821333448092142
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,7,0.008816000074148178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,3,0.00891733355820179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,15,0.009162666896979014
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,15,0.008634666601816813
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,31,0.008752000207702318
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,63,0.008992000172535578
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,63,0.008778666456540426
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,127,0.009039999917149544
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,127,0.008602666358153025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,255,0.008976000050703684
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,255,0.008837333569924036
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,511,0.012261333564917246
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,31,0.006677333265542984
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,511,0.010741333166758219
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,1023,0.026586666703224182
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,2047,0.027488000690937042
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,2047,0.04481600224971771
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,4095,0.08166933556397755
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,1,0.012373333175977072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,1,0.010735999792814255
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,4095,0.04576000074545542
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,3,0.01240533341964086
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,3,0.010863999525705973
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,7,0.011754666765530905
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,7,0.010746666540702185
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,1,0.00873066671192646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,15,0.012522666404644648
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,15,0.010794666906197866
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,3,0.008890666688481966
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,31,0.011418666690587997
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,31,0.008645333349704742
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,31,0.01109333336353302
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,63,0.012389333297808966
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,63,0.010762666662534079
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,127,0.013173333058754602
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,127,0.010890666395425797
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,255,0.012794667234023413
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,255,0.010746666540702185
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,1023,0.04520000020662943
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,1023,0.027429332335789997
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,511,0.012869333227475485
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,511,0.027050666511058807
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,2047,0.08248533308506012
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,1,0.006735999758044879
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,2047,0.04655999938646952
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,1023,0.013050666699806849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,3,0.006810666372378667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,4095,0.08262399832407634
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,7,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,1,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,15,0.007071999832987785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,31,0.006810666372378667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,7,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,31,0.006778666749596596
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,63,0.007045333584149678
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,63,0.0069226666043202085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,127,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,255,0.006981333096822103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,255,0.006688000013430913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,511,0.008885333314538002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,511,0.008682666967312494
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,1023,0.00897066667675972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,1023,0.008901333436369896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,2047,0.01033599985142549
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,4095,0.008922666932145754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,4095,0.008986666798591614
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,8191,0.012970666090647379
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,8191,0.012863999853531519
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,16383,0.015157333264748255
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,16383,0.014864000181357065
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,32767,0.03149333347876867
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,1,0.006837333242098491
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,1,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,3,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,3,0.006741333131988843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,7,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,7,0.006538666784763336
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,15,0.00702400008837382
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,3,0.007002666592597961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,15,0.006511999915043513
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,15,0.007082666580875714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,31,0.0069226666043202085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,31,0.006682666639486949
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,63,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,63,0.006826666494210561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,127,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,127,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,127,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,255,0.006784000123540561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,255,0.006800000245372455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,511,0.008714666590094566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,1023,0.008858666444818178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,2047,0.009002666920423508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,2047,0.008762666955590248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,2047,0.008613333106040955
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,4095,0.010853332777818045
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,4095,0.010821333775917688
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,8191,0.013765333841244379
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,8191,0.012826666235923767
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,16383,0.015008000036080679
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,16383,0.02938133229811986
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,32767,0.04748799900213877
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,32767,0.029440000653266907
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,1,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,1,0.0069226666043202085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,32767,0.017914666483799618
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,3,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,7,0.00684799998998642
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,7,0.006821333120266597
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,15,0.006597333277265231
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,15,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,31,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,31,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,63,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,127,0.006831999868154526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,127,0.007397333160042763
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,255,0.006858666737874349
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,255,0.007167999943097432
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,511,0.00891733355820179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,511,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,1023,0.00891733355820179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,1023,0.009056000038981438
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,4095,0.15450666348139444
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,2047,0.01089599976936976
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,4095,0.010618666807810465
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,4095,0.013130666067202887
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,511,0.007146666447321574
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,8191,0.03595199932654699
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,8191,0.013210666676362356
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,1023,0.008592000231146812
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,16383,0.054154664278030396
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,16383,0.02754133443037669
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,32767,0.04541333516438802
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,32767,0.08052266637484233
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,1,0.010858666151762009
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,1,0.00891733355820179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,3,0.010949333508809408
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,3,0.009098666409651438
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,7,0.010784000158309937
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,3,0.006730666384100914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,7,0.009066666786869368
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,15,0.010960000256697336
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,15,0.008858666444818178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,31,0.010650667051474253
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,31,0.009056000038981438
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,63,0.010816000401973724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,63,0.00707733320693175
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,63,0.00926399976015091
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,127,0.008938666433095932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,127,0.010853332777818045
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,255,0.010826667149861654
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,255,0.00902399979531765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,511,0.013173333058754602
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,2047,0.009072000160813332
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,511,0.011034666250149408
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,1023,0.027056001126766205
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,1023,0.015173333386580149
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,1,0.012576000144084295
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,3,0.012576000144084295
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,2047,0.04543999830881754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,2047,0.0284853329261144
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,7,0.011071999867757162
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,3,0.010757333288590113
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,15,0.011744000017642975
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,15,0.010816000401973724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,7,0.011685332904259363
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,63,0.012773333738247553
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,63,0.010735999792814255
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,127,0.012416000167528788
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,127,0.010751999914646149
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,255,0.01249066616098086
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,255,0.010869332899649939
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,31,0.012981332838535309
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,31,0.010911999891201654
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,511,0.013424000392357508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,511,0.027056001126766205
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,1,0.01098666712641716
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,2047,0.04576000074545542
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,2047,0.0823520024617513
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,1,0.01687466725707054
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,1023,0.02789866675933202
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,3,0.015072000523408255
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,1,0.015168000012636185
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,7,0.016986666868130367
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,7,0.015040000279744467
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,3,0.017114666601022083
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,15,0.01693333312869072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,31,0.014943999548753103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,31,0.01725333308180173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,63,0.016917333006858826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,63,0.014858666807413101
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,127,0.01682666689157486
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,127,0.014773332824309668
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,255,0.015146666516860327
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,255,0.02661866694688797
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,1023,0.04647466540336609
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,1023,0.08028266827265422
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,511,0.05628266433874766
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,511,0.027786667148272198
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,1023,0.04577066500981649
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,2047,0.08359466989835103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,2047,0.15384533007939658
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,1,0.014858666807413101
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,1,0.012762666990359625
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,15,0.015077333897352219
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,31,0.014762666076421738
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,15,0.015082667271296183
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,3,0.01479999969402949
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,3,0.012986666212479273
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,7,0.01470400020480156
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,31,0.013050666699806849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,63,0.014842666685581207
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,63,0.012757333616415659
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,127,0.014981333166360855
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,127,0.012858666479587555
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,255,0.012768000364303589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,511,0.027301333844661713
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,1,0.01692266638080279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,511,0.017050666113694508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,1,0.01481066644191742
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,7,0.012746666868527731
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,3,0.017162666966517765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,7,0.017050666113694508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,3,0.014682666709025701
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,15,0.01700266698996226
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,7,0.01515199989080429
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,15,0.015008000036080679
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,31,0.0170666662355264
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,31,0.014752000570297241
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,255,0.014725333700577417
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,63,0.01695466662446658
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,63,0.014709333578745524
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,127,0.01711999997496605
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,127,0.014981333166360855
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,1,0.02741866558790207
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,255,0.014767999450365702
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,511,0.04539733131726583
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,15,0.012709333250919977
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,511,0.027802666028340656
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,1,0.0234400009115537
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,3,0.027317332724730175
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,3,0.02309333284695943
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,7,0.027098665634791057
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,15,0.027429332335789997
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,31,0.02771199991305669
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,31,0.023045333723227184
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,7,0.02290133386850357
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,15,0.02311466634273529
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,63,0.027119999130566914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,63,0.023226665953795116
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,255,0.02733866622050603
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,127,0.02956266701221466
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,127,0.02309333284695943
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,255,0.04568533102671305
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,1,0.021546666820844013
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,1,0.01700266698996226
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,3,0.02120000123977661
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,511,0.046480000019073486
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,511,0.08183999856313069
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,3,0.017221332838137943
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,7,0.022645334402720135
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,7,0.017162666966517765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,15,0.021253332495689392
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,15,0.017050666113694508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,255,0.02779199928045273
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,63,0.017136000096797943
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,31,0.01714133347074191
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,31,0.022602667411168415
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,127,0.02141333371400833
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,127,0.017157333592573803
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,255,0.016943999876578648
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,255,0.027984000742435455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,1,0.027061333258946735
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,63,0.022677332162857056
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,1,0.02298133323589961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,3,0.02717333287000656
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,3,0.023247999449570973
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,7,0.027269333600997925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,7,0.023061332603295643
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,15,0.027306665976842243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,15,0.023210667073726654
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,31,0.02718399961789449
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,31,0.023205332458019257
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,63,0.02714666724205017
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,63,0.02327999969323476
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,127,0.029813334345817566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,127,0.02334933231274287
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,255,0.04577066500981649
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,255,0.028933333853880566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,1,0.04961599906285604
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,1,0.03962666789690653
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,3,0.04924799998601278
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,3,0.03972800076007843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,7,0.04775466521581014
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,7,0.039408000806967415
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,31,0.03966933240493139
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,31,0.048997332652409874
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,63,0.04929600159327189
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,63,0.039594667653242745
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,15,0.03975466638803482
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,15,0.048714667558670044
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,3,0.007018666714429855
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,1,0.0069973332186539965
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,3,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,7,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,7,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,15,0.007887999837597212
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,1,0.006831999868154526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,15,0.0068693334857622785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,31,0.0069866664707660675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,31,0.007135999699433644
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,63,0.007167999943097432
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,63,0.006613333399097125
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,127,0.050928001602490745
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,127,0.007029333462317784
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,127,0.041562666495641075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,127,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,255,0.007151999821265538
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,255,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,511,0.008746666833758354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,1023,0.00898133342464765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,2047,0.008943999807039896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,4095,0.012725333372751871
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,4095,0.010992000500361124
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,8191,0.013002666334311167
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,8191,0.015087999403476715
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,16383,0.030037333567937214
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,511,0.008832000195980072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,1023,0.00878399983048439
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,16383,0.016858667135238647
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,32767,0.04821866750717163
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,32767,0.03142933299144109
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,1,0.00707733320693175
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,1,0.006746666505932808
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,3,0.00696000022192796
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,3,0.006735999758044879
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,7,0.006751999879876773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,7,0.006762666627764702
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,255,0.04891733328501383
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,15,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,255,0.08876267075538635
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,63,0.006757333253820737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,63,0.006730666384100914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,127,0.006789333497484525
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,255,0.006784000123540561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,255,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,511,0.00879466657837232
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,511,0.007034666836261749
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,127,0.007029333462317784
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,1023,0.008954666554927826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,1023,0.008874666566650072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,2047,0.010741333166758219
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,2047,0.008762666955590248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,4095,0.012917333592971167
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,4095,0.010773333410422007
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,8191,0.02696000039577484
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,8191,0.012847999731699625
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,16383,0.04543999830881754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,16383,0.027002667387326557
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,2047,0.008992000172535578
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,32767,0.08105066418647766
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,32767,0.04554666578769684
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,1,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,1,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,3,0.0069866664707660675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,3,0.006831999868154526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,7,0.006709333509206772
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,7,0.007055999711155891
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,15,0.007178666690985362
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,15,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,31,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,31,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,15,0.007087999954819679
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,63,0.006709333509206772
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,63,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,31,0.006714666883150737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,127,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,127,0.006768000001708667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,255,0.007061333085099856
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,511,0.008890666688481966
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,511,0.00707733320693175
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,1023,0.009018666421373686
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,1023,0.008879999940594038
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,2047,0.011114666859308878
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,2047,0.010938666760921478
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,4095,0.012794667234023413
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,4095,0.025978667040665943
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,8191,0.027034667630990345
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,8191,0.04481600224971771
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,16383,0.08053333560625713
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,16383,0.045466666420300804
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,32767,0.1527253290017446
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,32767,0.08116800089677174
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,31,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,255,0.006730666384100914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,1,0.03760000069936117
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,1,0.027402666707833607
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,3,0.03766400118668874
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,7,0.03766400118668874
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,3,0.02734400083621343
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,7,0.0269813338915507
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,15,0.03736533224582672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,15,0.02714666724205017
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,31,0.037605332831541695
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,63,0.03738666574160258
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,31,0.027237333357334137
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,63,0.027301333844661713
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,127,0.02720000098148982
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,127,0.03740799923737844
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,1,0.04775466521581014
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,1,0.0396373321612676
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,3,0.039706667264302574
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,3,0.04833066463470459
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,7,0.04773333172003428
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,7,0.03969600051641464
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,15,0.04894400139649709
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,15,0.039647998909155525
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,31,0.04797333478927612
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,31,0.03938666731119156
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,63,0.04779199759165446
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,63,0.03965866565704346
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,1,0.0888320008913676
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,127,0.05180266499519348
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,127,0.04155199974775314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,1,0.07225066423416138
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,3,0.08845333258310954
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,3,0.07242133220036824
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,7,0.0888853371143341
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,15,0.08807999889055888
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,7,0.07253333429495494
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,31,0.07230400045712788
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,31,0.08886933326721191
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,63,0.08889066179593404
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,15,0.07217599948247273
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,63,0.07294400036334991
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,127,0.09251733620961507
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,127,0.07235200206438701
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,float16,1,0.06929066777229309
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,1,0.04775999983151754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,float16,3,0.06974933544794719
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,3,0.047653332352638245
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,float16,7,0.06982933481534322
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,7,0.047695999344189964
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,float16,15,0.06994666655858357
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,15,0.04764266808827718
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,float16,31,0.06820266445477803
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,31,0.047184000412623085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,float16,63,0.06834666430950165
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,float16,1,0.08854400118192036
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,63,0.04780266682306925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,1,0.07239466905593872
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,float16,3,0.0885706643263499
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,3,0.07275733351707458
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,float16,7,0.08875200152397156
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,7,0.07441600163777669
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,float16,15,0.08880533774693807
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,15,0.07251200079917908
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,float16,31,0.08861333131790161
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,31,0.07262933254241943
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,float16,63,0.08773866295814514
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,63,0.07253866891066234
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,float16,1,0.16729066769282022
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,1,0.1361120045185089
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,3,0.13699733217557272
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,float16,7,0.1672160029411316
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,7,0.1362826625506083
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,1,0.007391999786098798
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,1,0.007034666836261749
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,3,0.007242666557431221
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,3,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,7,0.007584000006318092
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,float16,3,0.16805867354075113
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,7,0.006720000257094701
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,15,0.007696000238259633
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,15,0.006762666627764702
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,31,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,31,0.00697066696981589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,63,0.00701333334048589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,63,0.007050666958093643
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,127,0.0068693334857622785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,15,0.1372213363647461
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,127,0.007007999966541926
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,255,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,255,0.007029333462317784
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,511,0.008581333483258883
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,511,0.008965333302815756
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,1023,0.008885333314538002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,1023,0.00873066671192646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,2047,0.011018666128317514
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,2047,0.009088000282645226
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,4095,0.012645332763592402
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,4095,0.011066666493813196
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,8191,0.014965333044528961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,8191,0.027066667874654133
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,16383,0.045925334095954895
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,1,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,1,0.006714666883150737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,3,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,3,0.006618666773041089
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,7,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,7,0.006821333120266597
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,15,0.007109333450595538
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,float16,15,0.1675893266995748
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,31,0.1360373298327128
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,float16,31,0.16666666666666666
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,31,0.006698666761318843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,127,0.006666666517655055
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,float16,63,0.16800000270207724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,63,0.00702400008837382
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,63,0.006981333096822103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,255,0.006805333619316419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,511,0.008842666943868002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,511,0.008240000034372011
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,1023,0.009397333487868309
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,16383,0.028016000986099243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,1023,0.008746666833758354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,2047,0.010672000547250112
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,2047,0.01101333275437355
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,255,0.006725333631038666
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,4095,0.027290667096773785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,4095,0.012741333494583765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,8191,0.04541866481304169
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,8191,0.02703999976317088
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,16383,0.08097599943478902
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,15,0.006645333642760913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,1,0.006581333155433337
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,1,0.008447999755541483
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,31,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,3,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,7,0.007018666714429855
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,7,0.008650666723648706
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,15,0.008613333106040955
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,15,0.007002666592597961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,31,0.006655999769767125
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,63,0.008661333471536636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,63,0.0069973332186539965
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,127,0.008714666590094566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,127,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,127,0.006655999769767125
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,255,0.008656000097592672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,255,0.006720000257094701
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,511,0.00874133345981439
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,511,0.008496000121037165
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,1023,0.010784000158309937
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,2047,0.012826666235923767
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,1023,0.009050666665037474
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,2047,0.026026666164398193
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,4095,0.054645334680875145
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,4095,0.02641066660483678
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,8191,0.08083199958006541
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,8191,0.044954667488733925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,16383,0.04650133351484934
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,16383,0.1520639955997467
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,1,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,1,0.006826666494210561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,16383,0.0809440016746521
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,3,0.007861333588759104
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,63,0.13590400417645773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,3,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,7,0.007082666580875714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,15,0.006837333242098491
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,31,0.007605333502093951
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,15,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,31,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,63,0.006751999879876773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,127,0.007093333328763644
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,63,0.008101333553592363
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,255,0.006837333242098491
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,255,0.006800000245372455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,511,0.009050666665037474
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,511,0.00897066667675972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,31,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,1023,0.008922666932145754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,1023,0.00985599992175897
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,2047,0.01099733387430509
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,2047,0.00984533317387104
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,4095,0.012810666114091873
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,4095,0.011152000476916632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,8191,0.027269333600997925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,8191,0.015077333897352219
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,16383,0.045850664377212524
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,16383,0.02921066681543986
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,1,0.006773333375652631
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,1,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,3,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,3,0.006730666384100914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,7,0.0068693334857622785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,7,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,15,0.007050666958093643
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,31,0.0069226666043202085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,31,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,63,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,63,0.007055999711155891
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,127,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,3,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,255,0.007018666714429855
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,255,0.006640000268816948
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,7,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,511,0.008789333204428354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,511,0.006800000245372455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,1023,0.009290666629870733
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,1023,0.00884799969693025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,2047,0.011002667248249054
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,127,0.007002666592597961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,2047,0.010725333044926325
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,4095,0.026975999275843304
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,4095,0.012965332716703415
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,8191,0.04560000201066335
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,8191,0.0276053324341774
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,16383,0.08272000153859456
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,1,0.008736000085870424
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,1,0.007040000210205714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,16383,0.04582933088143667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,3,0.008602666358153025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,7,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,7,0.008410666758815447
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,15,0.008890666688481966
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,15,0.0069919998447100324
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,31,0.008682666967312494
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,31,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,63,0.007018666714429855
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,127,0.008714666590094566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,127,0.006981333096822103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,255,0.008656000097592672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,255,0.006800000245372455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,511,0.009098666409651438
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,511,0.00877333308259646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,15,0.006757333253820737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,1023,0.010751999914646149
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,2047,0.025914666553338368
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,2047,0.012863999853531519
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,4095,0.04404800136884054
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,4095,0.027109332382678986
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,8191,0.080335999528567
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,8191,0.04478399952252706
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,16383,0.08196799953778584
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,16383,0.15254933635393778
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,1,0.00916800027092298
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,1,0.008757333581646284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,3,0.008736000085870424
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,7,0.009077333534757296
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,7,0.008602666358153025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,3,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,15,0.008992000172535578
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,15,0.00877333308259646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,31,0.008762666955590248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,63,0.008576000109314919
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,31,0.008896000062425932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,63,0.009002666920423508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,63,0.008703999842206636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,127,0.008799999952316284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,127,0.008965333302815756
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,255,0.00915733352303505
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,255,0.00871999996403853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,511,0.011589333415031433
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,511,0.010693332801262537
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,1023,0.0262773334980011
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,1023,0.012762666990359625
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,127,0.00684799998998642
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,2047,0.026949333647886913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,4095,0.080335999528567
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,4095,0.04562133550643921
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,8191,0.1537866691748301
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,8191,0.08164266745249431
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,1,0.00697066696981589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,3,0.008938666433095932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,1,0.0069919998447100324
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,16383,0.29841599861780804
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,16383,0.15607999761899313
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,3,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,7,0.006821333120266597
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,7,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,15,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,31,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,31,0.0069226666043202085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,63,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,63,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,127,0.007029333462317784
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,127,0.006720000257094701
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,255,0.0069919998447100324
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,255,0.007173333317041397
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,511,0.008661333471536636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,511,0.008586666857202848
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,1023,0.009093333035707474
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,1023,0.008714666590094566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,2047,0.009029333169261614
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,2047,0.008703999842206636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,4095,0.008736000085870424
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,4095,0.008639999975760778
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,8191,0.010794666906197866
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,8191,0.009061333412925402
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,1023,0.008922666932145754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,16383,0.012778667112191519
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,32767,0.016938666502634685
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,65535,0.02515200028816859
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,65535,0.025034666061401367
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,32767,0.017130666722853977
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,1,0.006789333497484525
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,1,0.006533333410819371
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,3,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,7,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,7,0.006618666773041089
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,15,0.006725333631038666
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,2047,0.044079999128977455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,15,0.0068853336075941724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,31,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,31,0.006517333288987477
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,63,0.006773333375652631
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,127,0.006586666529377301
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,127,0.006645333642760913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,255,0.006757333253820737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,255,0.00666133314371109
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,511,0.008656000097592672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,511,0.007093333328763644
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,1023,0.008656000097592672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,1023,0.007376000285148621
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,2047,0.008922666932145754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,2047,0.008661333471536636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,4095,0.009061333412925402
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,15,0.006831999868154526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,8191,0.010309333602587381
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,8191,0.009093333035707474
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,16383,0.012917333592971167
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,16383,0.012815999488035837
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,32767,0.017445333302021027
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,32767,0.016997333616018295
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,65535,0.02126399924357732
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,65535,0.019173332800467808
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,1,0.006538666784763336
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,1,0.007141333073377609
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,3,0.006618666773041089
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,3,0.0064853330453236895
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,7,0.006693333387374878
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,7,0.006773333375652631
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,15,0.006629333520929019
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,15,0.006858666737874349
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,31,0.006629333520929019
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,31,0.006762666627764702
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,63,0.007104000076651573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,63,0.006746666505932808
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,127,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,127,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,3,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,255,0.006800000245372455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,255,0.006730666384100914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,511,0.006693333387374878
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,1023,0.00891733355820179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,63,0.007071999832987785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,1023,0.007285333548982938
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,2047,0.00884799969693025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,2047,0.008559999987483025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,4095,0.008736000085870424
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,4095,0.008842666943868002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,8191,0.010133333504199982
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,8191,0.00897066667675972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,3,0.00696000022192796
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,16383,0.013088000317414602
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,32767,0.017194667210181553
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,32767,0.015504000087579092
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,65535,0.017301333447297413
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,1,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,1,0.00666133314371109
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,3,0.006634666894872983
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,3,0.006671999891599019
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,7,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,7,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,16383,0.01310933381319046
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,15,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,15,0.006746666505932808
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,31,0.006784000123540561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,31,0.006826666494210561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,63,0.006575999781489372
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,63,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,127,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,511,0.008874666566650072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,255,0.0069386667261521024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,255,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,511,0.00891733355820179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,511,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,1023,0.00890666681031386
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,1023,0.008282666405042013
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,2047,0.008752000207702318
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,2047,0.008965333302815756
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,4095,0.008767999708652496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,16383,0.013408000270525614
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,8191,0.011365332951148352
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,8191,0.012714666624863943
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,4095,0.008602666358153025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,65535,0.033157333731651306
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,16383,0.013088000317414602
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,16383,0.014975999792416891
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,32767,0.0164533331990242
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,32767,0.036320000886917114
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,65535,0.031343999008337654
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,65535,0.05436266462008158
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,1,0.007162666569153468
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,3,0.007018666714429855
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,3,0.006757333253820737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,7,0.00702400008837382
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,15,0.006826666494210561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,7,0.007018666714429855
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,15,0.006981333096822103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,31,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,63,0.0068853336075941724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,127,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,127,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,255,0.006858666737874349
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,255,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,511,0.008821333448092142
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,511,0.00784533346692721
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,1023,0.009045333291093508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,1023,0.008943999807039896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,2047,0.008863999818762144
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,2047,0.008767999708652496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,4095,0.008943999807039896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,4095,0.008879999940594038
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,8191,0.010693332801262537
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,8191,0.009125333279371262
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,16383,0.014981333166360855
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,16383,0.012858666479587555
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,32767,0.017829333742459614
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,32767,0.020021333048741024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,65535,0.022069332500298817
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,127,0.006597333277265231
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,65535,0.021151999632517498
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,1,0.006762666627764702
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,3,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,7,0.006640000268816948
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,7,0.006831999868154526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,15,0.006624000146985054
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,15,0.006544000158707301
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,31,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,4095,0.008789333204428354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,31,0.0068693334857622785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,63,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,63,0.006544000158707301
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,127,0.006704000135262807
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,255,0.006751999879876773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,255,0.006624000146985054
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,1,0.00679466687142849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,511,0.00867733359336853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,511,0.006682666639486949
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,1023,0.008752000207702318
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,1023,0.007407999907930692
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,31,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,63,0.007045333584149678
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,2047,0.008922666932145754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,2047,0.008565333361426989
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,4095,0.008703999842206636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,4095,0.008714666590094566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,8191,0.011007999380429586
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,8191,0.008997333546479544
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,16383,0.01470400020480156
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,16383,0.013093333691358566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,32767,0.015498666713635126
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,65535,0.031397332747777305
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,65535,0.01732800031701724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,1,0.006762666627764702
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,1,0.00666133314371109
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,3,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,3,0.006666666517655055
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,7,0.006671999891599019
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,7,0.00697066696981589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,15,0.00666133314371109
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,15,0.006655999769767125
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,31,0.006735999758044879
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,1,0.006858666737874349
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,31,0.006773333375652631
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,3,0.006778666749596596
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,63,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,63,0.006629333520929019
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,127,0.006671999891599019
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,127,0.006768000001708667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,255,0.007093333328763644
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,255,0.006709333509206772
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,511,0.008698666468262672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,511,0.007018666714429855
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,1023,0.008821333448092142
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,1023,0.008805333326260248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,2047,0.008682666967312494
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,2047,0.008672000219424566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,4095,0.008912000184257826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,4095,0.00890666681031386
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,8191,0.011584000041087469
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,8191,0.01101333275437355
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,16383,0.015141333142916361
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,127,0.006837333242098491
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,32767,0.030623999734719593
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,65535,0.04822400212287903
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,1,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,65535,0.03133866687615713
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,1,0.006575999781489372
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,3,0.007104000076651573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,3,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,7,0.0069919998447100324
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,7,0.006613333399097125
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,15,0.007040000210205714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,15,0.006688000013430913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,31,0.006629333520929019
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,63,0.007055999711155891
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,63,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,127,0.007034666836261749
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,127,0.0069226666043202085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,32767,0.017093333105246227
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,255,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,255,0.006666666517655055
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,511,0.008816000074148178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,1023,0.00855466661353906
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,2047,0.008592000231146812
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,2047,0.008762666955590248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,4095,0.01090666651725769
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,4095,0.010656000425418219
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,8191,0.013770667215188345
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,8191,0.012741333494583765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,16383,0.03833599885304769
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,16383,0.015024000157912573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,32767,0.057760000228881836
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,32767,0.029152000943819683
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,65535,0.08936533331871033
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,65535,0.04751466711362203
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,1,0.007226666435599327
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,16383,0.012709333250919977
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,32767,0.015311999867359797
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,1,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,3,0.008762666955590248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,3,0.007029333462317784
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,7,0.007050666958093643
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,7,0.006837333242098491
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,15,0.007786666974425316
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,15,0.00697066696981589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,31,0.007471999774376552
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,31,0.007007999966541926
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,31,0.006720000257094701
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,63,0.007584000006318092
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,63,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,127,0.008538666491707167
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,127,0.006757333253820737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,255,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,255,0.0069919998447100324
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,511,0.00903466654320558
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,511,0.00892800030608972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,511,0.007071999832987785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,1023,0.010656000425418219
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,1023,0.008789333204428354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,1023,0.008992000172535578
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,2047,0.012896000097195307
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,2047,0.011039999624093374
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,8191,0.04571733375390371
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,1,0.00890666681031386
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,8191,0.028346667687098186
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,4095,0.014661333213249842
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,1,0.007162666569153468
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,3,0.00879466657837232
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,3,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,7,0.008661333471536636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,7,0.007029333462317784
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,15,0.006741333131988843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,15,0.008645333349704742
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,31,0.008693333094318708
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,31,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,63,0.00707733320693175
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,63,0.008922666932145754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,127,0.008810666700204214
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,127,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,255,0.00860799973209699
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,255,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,511,0.009093333035707474
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,511,0.00895999992887179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,1023,0.011055999745925268
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,1023,0.009029333169261614
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,4095,0.045466666420300804
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,2047,0.012762666990359625
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,8191,0.08198399841785431
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,8191,0.045519997676213585
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,1,0.008901333436369896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,1,0.00867733359336853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,3,0.009061333412925402
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,3,0.008639999975760778
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,7,0.009088000282645226
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,7,0.0085333331177632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,15,0.00874133345981439
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,4095,0.027317332724730175
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,15,0.009535999968647957
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,31,0.00892800030608972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,63,0.00902399979531765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,63,0.008757333581646284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,127,0.008746666833758354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,255,0.009136000027259191
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,127,0.008896000062425932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,255,0.008645333349704742
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,511,0.012138667205969492
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,1023,0.026885333160559338
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,2047,0.0440586656332016
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,2047,0.027141332626342773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,2047,0.027317332724730175
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,4095,0.027269333600997925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,4095,0.08126399914423625
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,8191,0.1539413332939148
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,1,0.012703999876976013
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,1,0.010832000523805618
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,8191,0.08268799881140391
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,3,0.012901333471139273
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,7,0.012400000045696894
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,31,0.00877333308259646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,15,0.012442667037248611
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,15,0.011168000598748526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,31,0.01139733319481214
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,31,0.010842667271693548
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,63,0.013023999830087027
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,63,0.010693332801262537
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,511,0.01073066641887029
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,1023,0.013050666699806849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,127,0.012837332983811697
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,127,0.010874666273593903
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,255,0.011786667009194693
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,4095,0.04529066880544027
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,511,0.03543466577927271
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,511,0.013738666971524557
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,1023,0.04530666768550873
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,1023,0.027034667630990345
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,2047,0.08785067001978557
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,2047,0.0458186666170756
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,4095,0.08266666531562805
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,3,0.010954666882753372
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,1,0.007135999699433644
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,4095,0.1564586659272512
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,7,0.01097600037852923
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,1,0.006784000123540561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,3,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,3,0.006768000001708667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,8191,0.30590399106343585
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,7,0.007055999711155891
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,7,0.00707733320693175
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,15,0.007050666958093643
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,8191,0.1547040045261383
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,63,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,31,0.006858666737874349
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,63,0.007071999832987785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,127,0.007104000076651573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,255,0.0069866664707660675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,127,0.006682666639486949
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,511,0.008736000085870424
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,1023,0.009050666665037474
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,1023,0.009018666421373686
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,2047,0.008992000172535578
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,2047,0.008736000085870424
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,4095,0.009088000282645226
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,4095,0.009002666920423508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,8191,0.010773333410422007
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,8191,0.009141333401203156
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,255,0.010650667051474253
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,16383,0.015146666516860327
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,16383,0.01312000056107839
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,32767,0.017173333714405697
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,65535,0.03288000077009201
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,32767,0.019007999449968338
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,65535,0.018981333822011948
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,1,0.007040000210205714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,1,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,3,0.006698666761318843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,3,0.006671999891599019
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,7,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,7,0.006655999769767125
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,15,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,15,0.006650666395823161
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,31,0.006698666761318843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,31,0.006698666761318843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,63,0.006831999868154526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,63,0.006725333631038666
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,127,0.006720000257094701
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,127,0.007007999966541926
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,255,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,255,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,511,0.00867733359336853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,511,0.0069386667261521024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,1023,0.00867733359336853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,1023,0.008826666822036108
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,31,0.006800000245372455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,2047,0.00901333304742972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,2047,0.008645333349704742
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,4095,0.008890666688481966
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,4095,0.008879999940594038
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,255,0.007018666714429855
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,8191,0.012805332740147909
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,8191,0.010847999403874079
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,16383,0.015109332899252573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,16383,0.013125333935022354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,32767,0.02938666691382726
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,32767,0.015791999797026317
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,65535,0.04789333542188009
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,65535,0.03057066599527995
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,1,0.006693333387374878
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,3,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,1,0.006709333509206772
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,3,0.0069866664707660675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,7,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,7,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,15,0.007040000210205714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,15,0.006650666395823161
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,31,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,31,0.006693333387374878
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,63,0.00660800002515316
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,127,0.006714666883150737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,127,0.007141333073377609
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,255,0.006597333277265231
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,255,0.006709333509206772
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,511,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,511,0.00873066671192646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,1023,0.008874666566650072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,1023,0.008816000074148178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,2047,0.008746666833758354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,2047,0.008698666468262672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,4095,0.01007466639081637
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,8191,0.012698666503032049
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,8191,0.014757333944241205
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,16383,0.02900800108909607
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,16383,0.015103999525308609
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,32767,0.047135998805363975
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,32767,0.02937600016593933
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,65535,0.08274133503437042
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,65535,0.04832000037034353
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,1,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,3,0.00697066696981589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,3,0.006837333242098491
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,7,0.00701333334048589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,7,0.006746666505932808
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,15,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,15,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,15,0.006640000268816948
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,31,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,31,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,511,0.008170666793982187
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,63,0.006671999891599019
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,127,0.006858666737874349
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,127,0.006826666494210561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,255,0.006709333509206772
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,255,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,511,0.00871999996403853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,511,0.006735999758044879
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,1023,0.009056000038981438
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,63,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,2047,0.010832000523805618
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,2047,0.008842666943868002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,4095,0.012773333738247553
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,4095,0.010709332923094431
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,8191,0.01303999995191892
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,8191,0.03655466685692469
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,4095,0.010768000036478043
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,16383,0.05563733478387197
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,16383,0.02712533374627431
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,32767,0.08050133287906647
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,32767,0.045509333411852516
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,65535,0.08317866424719493
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,65535,0.15196266770362854
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,1,0.00706666645904382
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,1,0.008816000074148178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,1,0.008250666782259941
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,3,0.008650666723648706
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,3,0.0080960001796484
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,7,0.008863999818762144
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,7,0.007274666801095009
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,63,0.00684799998998642
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,15,0.00854399986565113
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,15,0.0069386667261521024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,31,0.008832000195980072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,63,0.008901333436369896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,63,0.008778666456540426
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,127,0.008559999987483025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,1023,0.008821333448092142
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,127,0.00697066696981589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,255,0.0086666668454806
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,255,0.007061333085099856
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,511,0.009359999870260557
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,511,0.008976000050703684
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,1023,0.012725333372751871
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,1023,0.010751999914646149
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,2047,0.027082666754722595
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,2047,0.012960000584522883
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,1,0.008912000184257826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,1,0.008725333337982496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,4095,0.045514668027559914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,3,0.008976000050703684
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,4095,0.027061333258946735
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,7,0.008703999842206636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,7,0.008922666932145754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,3,0.00878399983048439
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,15,0.008656000097592672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,31,0.008496000121037165
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,31,0.00915733352303505
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,63,0.009072000160813332
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,63,0.008778666456540426
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,127,0.008896000062425932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,127,0.008746666833758354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,15,0.009061333412925402
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,255,0.008714666590094566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,255,0.008896000062425932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,511,0.011701333026091257
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,511,0.010762666662534079
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,1023,0.02722666660944621
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,1023,0.013061333447694778
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,4095,0.08261333405971527
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,4095,0.045781334241231285
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,2047,0.04555733501911163
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,1,0.010933333386977514
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,3,0.012736000120639801
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,3,0.010816000401973724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,7,0.012613333761692047
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,15,0.012847999731699625
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,15,0.010928000013033548
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,31,0.012485332787036896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,7,0.010714666297038397
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,31,0.010778666784365972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,63,0.012986666212479273
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,63,0.010656000425418219
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,31,0.009008000294367472
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,127,0.010741333166758219
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,127,0.01209066684047381
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,255,0.012853333105643591
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,255,0.010949333508809408
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,511,0.027136000494162243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,511,0.013354666531085968
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,2047,0.08228266735871632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,1,0.011413333316644033
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,2047,0.027653334041436512
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,1023,0.04565866788228353
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,1023,0.02716800073782603
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,2047,0.04669866462548574
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,1,0.014752000570297241
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,1,0.017194667210181553
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,4095,0.15571733315785727
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,7,0.015125333021084467
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,15,0.014901333798964819
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,15,0.01722666621208191
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,4095,0.08436800042788188
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,31,0.017114666601022083
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,31,0.015066667149464289
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,63,0.017242666333913803
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,63,0.015119999647140503
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,3,0.014869333555301031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,7,0.017237332959969837
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,127,0.014901333798964819
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,3,0.017173333714405697
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,127,0.016949333250522614
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,255,0.02994133283694585
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,1023,0.08052266637484233
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,1023,0.04568000137805939
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,2047,0.15197333693504333
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,511,0.05481066803137461
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,2047,0.08301333089669545
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,4095,0.15633599956830344
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,4095,0.2961866656939189
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,1,0.010735999792814255
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,1,0.009152000149091085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,3,0.01090666651725769
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,3,0.008992000172535578
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,7,0.010885333021481832
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,15,0.010842667271693548
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,255,0.015013333410024643
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,15,0.00919999989370505
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,31,0.010703999549150467
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,31,0.009242666885256767
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,511,0.02741866558790207
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,63,0.008837333569924036
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,127,0.010826667149861654
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,63,0.010784000158309937
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,127,0.009008000294367472
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,255,0.010645333677530289
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,511,0.012826666235923767
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,1,0.012639999389648438
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,511,0.011258666714032492
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,1023,0.014959999670584997
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,1023,0.027647999425729115
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,7,0.00884799969693025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,1,0.010794666906197866
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,3,0.012901333471139273
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,3,0.01180800050497055
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,7,0.012981332838535309
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,15,0.01101333275437355
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,7,0.010778666784365972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,15,0.012789333860079447
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,31,0.012170666207869848
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,31,0.010784000158309937
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,63,0.012133333832025528
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,63,0.01098666712641716
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,127,0.0122079998254776
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,255,0.00895999992887179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,127,0.010746666540702185
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,255,0.010879999647537867
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,255,0.012810666114091873
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,511,0.027093333502610523
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,511,0.013066666821638743
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,1,0.014805333067973455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,1,0.0170666662355264
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,1023,0.04573333263397217
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,1023,0.02739199995994568
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,3,0.015157333264748255
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,3,0.016864000509182613
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,7,0.01720000058412552
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,7,0.014917333920796713
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,15,0.01693333312869072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,15,0.015066667149464289
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,63,0.015589332828919092
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,63,0.01717866708834966
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,127,0.014842666685581207
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,127,0.01706133286158244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,255,0.02722666660944621
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,255,0.015098666151364645
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,31,0.016997333616018295
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,511,0.0452106644709905
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,511,0.027002667387326557
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,3,0.027237333357334137
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,1,0.027242665489514668
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,1,0.0232640008131663
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,1023,0.08124800026416779
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,1023,0.046426668763160706
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,7,0.023189333577950794
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,15,0.027248000105222065
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,15,0.023178666830062866
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,3,0.023317334552605946
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,31,0.014794666320085526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,7,0.02739733209212621
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,63,0.027477333943049114
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,63,0.022991999983787537
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,127,0.037151999771595
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,255,0.02815466622511546
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,255,0.04483733574549357
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,31,0.027461332579453785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,31,0.023157333334287006
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,1023,0.08228800197442372
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,511,0.08096000055472057
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,511,0.04577066500981649
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,1,0.01488000030318896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,127,0.023152001202106476
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,1,0.01267733300725619
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,3,0.014741333822409311
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,3,0.012863999853531519
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,15,0.01481066644191742
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,1023,0.1537440021832784
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,15,0.012853333105643591
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,31,0.014848000059525171
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,31,0.01259200026591619
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,7,0.014949332922697067
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,7,0.012821332861979803
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,63,0.01504533365368843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,63,0.012736000120639801
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,127,0.012853333105643591
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,127,0.015125333021084467
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,255,0.014730667074521383
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,255,0.0129120002190272
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,511,0.028378665447235107
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,1,0.017029333859682083
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,511,0.016016000260909397
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,1,0.014912000546852747
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,3,0.01709866647919019
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,3,0.014741333822409311
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,15,0.017136000096797943
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,7,0.017162666966517765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,7,0.014864000181357065
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,15,0.014954666296641031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,31,0.016917333006858826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,63,0.016949333250522614
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,31,0.015024000157912573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,63,0.015013333410024643
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,127,0.01479999969402949
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,127,0.01714666684468587
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,255,0.027306665976842243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,255,0.014922666052977243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,1,0.027450665831565857
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,511,0.027456000447273254
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,1,0.02295999974012375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,3,0.02749866743882497
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,3,0.023226665953795116
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,7,0.02738133321205775
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,7,0.023168000082174938
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,15,0.02734400083621343
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,31,0.02741333345572154
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,15,0.02311466634273529
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,31,0.022997332115968067
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,511,0.045781334241231285
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,63,0.027263998985290527
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,63,0.023178666830062866
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,127,0.02934933453798294
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,127,0.022944000860055287
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,255,0.045466666420300804
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,255,0.028031999866167705
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,1,0.0476693312327067
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,1,0.03963200002908707
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,511,0.04651199777921041
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,511,0.08192533254623413
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,3,0.04789333542188009
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,3,0.039664000272750854
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,15,0.04920533299446106
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,7,0.047882666190465294
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,15,0.03955733279387156
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,7,0.03931200007597605
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,63,0.04923733572165171
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,63,0.03957866628964742
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,127,0.051301335295041404
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,127,0.03937066594759623
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,31,0.039461334546407066
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,31,0.04869333406289419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,3,0.007040000210205714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,7,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,255,0.08205866813659668
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,7,0.006618666773041089
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,1,0.00706666645904382
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,15,0.006858666737874349
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,31,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,255,0.047728002071380615
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,63,0.006768000001708667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,127,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,127,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,255,0.00684799998998642
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,255,0.006586666529377301
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,511,0.00873066671192646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,511,0.008639999975760778
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,15,0.007141333073377609
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,63,0.006751999879876773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,1023,0.00898133342464765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,1023,0.008805333326260248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,2047,0.00903466654320558
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,2047,0.008912000184257826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,4095,0.008954666554927826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,4095,0.008997333546479544
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,8191,0.012725333372751871
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,3,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,8191,0.012917333592971167
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,511,0.15370666980743408
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,16383,0.01664000004529953
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,32767,0.03072533259789149
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,32767,0.017792000124851864
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,511,0.08266133566697438
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,1,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,65535,0.031093334158261616
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,1,0.00702400008837382
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,1,0.006810666372378667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,3,0.006602666651209195
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,7,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,7,0.006682666639486949
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,15,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,15,0.006730666384100914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,31,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,31,0.006693333387374878
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,63,0.007007999966541926
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,63,0.00666133314371109
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,127,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,127,0.007055999711155891
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,255,0.006735999758044879
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,255,0.006826666494210561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,511,0.00871999996403853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,511,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,1023,0.008896000062425932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,1023,0.008757333581646284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,2047,0.008672000219424566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,2047,0.008997333546479544
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,4095,0.010981333752473196
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,8191,0.012810666114091873
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,4095,0.010746666540702185
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,8191,0.014570667097965876
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,16383,0.029152000943819683
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,16383,0.01509333277742068
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,32767,0.04771199822425842
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,32767,0.029189333319664
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,65535,0.08443199594815572
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,65535,0.047728002071380615
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,1,0.007082666580875714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,65535,0.04964800179004669
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,1,0.006725333631038666
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,3,0.007114666824539502
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,3,0.0068853336075941724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,3,0.006714666883150737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,16383,0.014842666685581207
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,7,0.0069919998447100324
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,15,0.006751999879876773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,15,0.00684799998998642
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,31,0.00702400008837382
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,31,0.006565333033601443
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,63,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,63,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,127,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,127,0.006741333131988843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,255,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,511,0.008949333180983862
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,511,0.006789333497484525
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,1023,0.00898133342464765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,31,0.007045333584149678
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,1023,0.00878399983048439
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,2047,0.010992000500361124
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,2047,0.008997333546479544
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,4095,0.012778667112191519
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,4095,0.010693332801262537
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,8191,0.027104000250498455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,8191,0.013007999708255133
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,16383,0.04517866671085358
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,16383,0.027136000494162243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,32767,0.0812799980243047
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,32767,0.046112000942230225
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,1,0.006826666494210561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,65535,0.15320000052452087
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,65535,0.08330666522185008
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,3,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,3,0.006831999868154526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,7,0.006810666372378667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,15,0.006709333509206772
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,15,0.0069973332186539965
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,31,0.00702400008837382
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,31,0.006778666749596596
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,63,0.006618666773041089
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,63,0.007002666592597961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,127,0.006821333120266597
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,127,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,255,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,255,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,511,0.00878399983048439
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,511,0.007637333124876022
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,1023,0.008762666955590248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,1023,0.009583999713261923
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,7,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,2047,0.010757333288590113
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,2047,0.011823999385039011
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,4095,0.012965332716703415
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,4095,0.035616000493367515
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,8191,0.04698666433493296
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,8191,0.0269813338915507
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,16383,0.08499200145403545
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,16383,0.045738667249679565
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,32767,0.15291733543078104
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,65535,0.29703466097513836
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,65535,0.15365866820017496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,1,0.006714666883150737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,7,0.006837333242098491
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,255,0.006805333619316419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,32767,0.08145066599051158
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,1,0.021541332205136616
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,1,0.017093333105246227
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,3,0.023007998863856
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,7,0.021525333325068157
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,7,0.01729600007335345
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,15,0.021770666042963665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,15,0.017322666943073273
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,31,0.01708799973130226
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,31,0.021583999196688335
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,3,0.016858667135238647
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,63,0.022117334107557934
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,63,0.01703466723362605
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,127,0.022463999688625336
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,127,0.016895999511082966
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,1,0.0271573339899381
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,255,0.017077332983414333
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,1,0.023050665855407715
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,3,0.027424000203609467
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,3,0.023215999205907185
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,7,0.027327999472618103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,7,0.023007998863856
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,255,0.028698667883872986
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,15,0.027082666754722595
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,31,0.027376001079877216
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,15,0.023221333821614582
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,31,0.024330665667851765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,63,0.02737066646416982
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,63,0.023018665611743927
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,127,0.023039999107519787
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,127,0.03121600051720937
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,255,0.046485334634780884
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,1,0.047781333327293396
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,255,0.029322666426499683
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,1,0.039674667020638786
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,3,0.050000001986821495
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,3,0.039279999832312264
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,7,0.04961066444714864
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,7,0.03976000100374222
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,15,0.04773866633574168
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,15,0.039306665460268654
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,31,0.04937600096066793
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,31,0.03962666789690653
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,63,0.04808533191680908
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,63,0.03938133269548416
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,127,0.050517335534095764
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,127,0.041509332756201424
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,1,0.08785600463549297
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,255,0.0831573357184728
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,1,0.07250133156776428
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,255,0.04785599807898203
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,3,0.07250666618347168
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,3,0.08915733297665913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,7,0.07250666618347168
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,15,0.08820266524950664
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,31,0.08873066306114197
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,63,0.08880000313123067
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,63,0.07230400045712788
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,15,0.07246399919191997
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,7,0.08875200152397156
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,31,0.07235200206438701
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,127,0.09232532978057861
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,127,0.07215466598669688
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,255,0.08772266904513042
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,255,0.15576000014940897
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,1,0.03762666632731756
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,1,0.0271519993742307
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,3,0.03737066686153412
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,3,0.027322667340437572
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,7,0.027434666951497395
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,7,0.03766400118668874
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,31,0.02739733209212621
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,31,0.03775466730197271
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,15,0.037274666130542755
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,63,0.03732266773780187
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,15,0.02736533433198929
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,63,0.02731200059254964
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,127,0.027456000447273254
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,127,0.037647999823093414
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,1,0.04859733581542969
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,1,0.039546666045983635
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,3,0.0484746644894282
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,3,0.03975466638803482
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,7,0.04772266745567322
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,7,0.03951466580231985
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,15,0.04970133304595947
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,15,0.03949866692225138
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,31,0.04891733328501383
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,63,0.04880533119042715
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,63,0.039503999054431915
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,127,0.05229333539803823
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,127,0.04146133363246918
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,31,0.0396373321612676
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,1,0.0888266662756602
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,1,0.07232533395290375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,3,0.07240533332029979
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,7,0.08890666564305623
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,7,0.07248533268769582
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,15,0.08867200215657552
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,3,0.0888320008913676
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,15,0.07249066730340321
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,31,0.08790399630864461
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,31,0.07211199899514516
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,63,0.08879466851552327
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,63,0.07214400172233582
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,1,0.16662399967511496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,127,0.09319466352462769
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,127,0.07240533332029979
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,1,0.13687466581662497
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,3,0.13714667161305746
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,7,0.16702399651209512
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,3,0.1676266590754191
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,7,0.13699199755986533
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,15,0.1665493349234263
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,1,0.008010666817426682
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,1,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,3,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,3,0.006773333375652631
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,7,0.0069226666043202085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,7,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,31,0.13589866956075033
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,15,0.13618666927019754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,15,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,31,0.1674826741218567
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,31,0.006821333120266597
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,63,0.007957333077987036
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,127,0.006981333096822103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,127,0.006602666651209195
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,31,0.007194666812817256
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,255,0.007055999711155891
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,255,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,15,0.006768000001708667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,511,0.008597333605090777
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,63,0.007146666447321574
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,511,0.008832000195980072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,1023,0.009178666397929192
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,1023,0.008672000219424566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,63,0.16666666666666666
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,2047,0.00897066667675972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,2047,0.008949333180983862
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,4095,0.011535999675591787
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,4095,0.010960000256697336
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,8191,0.014949332922697067
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,8191,0.013050666699806849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,16383,0.016917333006858826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,32767,0.03141333411137263
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,1,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,1,0.006544000158707301
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,3,0.006773333375652631
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,7,0.007050666958093643
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,7,0.00666133314371109
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,16383,0.029296000798543293
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,15,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,15,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,31,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,31,0.0069919998447100324
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,63,0.00701333334048589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,63,0.006544000158707301
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,127,0.0069919998447100324
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,127,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,255,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,255,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,511,0.00890666681031386
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,511,0.006773333375652631
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,1023,0.008890666688481966
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,1023,0.008949333180983862
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,63,0.13607999682426453
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,2047,0.010666667173306147
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,2047,0.00873066671192646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,4095,0.012970666090647379
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,4095,0.010703999549150467
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,8191,0.013023999830087027
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,16383,0.04569066564242045
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,16383,0.027274665733178455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,32767,0.08246933420499165
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,32767,0.046538665890693665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,1,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,1,0.006735999758044879
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,8191,0.02735466758410136
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,32767,0.04785599807898203
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,3,0.006757333253820737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,7,0.007045333584149678
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,3,0.007055999711155891
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,7,0.006682666639486949
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,15,0.007093333328763644
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,31,0.007050666958093643
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,31,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,127,0.17384000619252524
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,63,0.00684799998998642
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,127,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,127,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,255,0.006826666494210561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,511,0.008992000172535578
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,511,0.007520000139872233
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,127,0.13774399956067404
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,1023,0.009610666582981745
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,1023,0.008816000074148178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,2047,0.011141333729028702
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,2047,0.010858666151762009
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,4095,0.013072000195582708
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,8191,0.04538666705290476
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,8191,0.027066667874654133
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,16383,0.08038400113582611
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,16383,0.04570133487383524
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,32767,0.15406399965286255
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,1,0.00884799969693025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,1,0.006762666627764702
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,3,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,3,0.009072000160813332
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,15,0.006826666494210561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,3,0.006784000123540561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,7,0.008938666433095932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,63,0.006810666372378667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,7,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,15,0.008805333326260248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,15,0.007082666580875714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,255,0.0069866664707660675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,31,0.007514666765928268
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,63,0.00874133345981439
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,127,0.008778666456540426
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,127,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,4095,0.027050666511058807
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,255,0.008650666723648706
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,255,0.007040000210205714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,511,0.008896000062425932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,511,0.008736000085870424
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,1023,0.008714666590094566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,1023,0.011039999624093374
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,2047,0.025792000194390614
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,4095,0.026677332818508148
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,4095,0.04890666902065277
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,8191,0.045109331607818604
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,8191,0.08044800162315369
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,16383,0.15187199910481772
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,16383,0.08099733293056488
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,32767,0.29660799105962116
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,1,0.0069919998447100324
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,32767,0.15332266688346863
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,3,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,3,0.006655999769767125
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,31,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,7,0.006981333096822103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,63,0.007045333584149678
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,7,0.00697066696981589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,15,0.006800000245372455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,31,0.006768000001708667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,63,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,63,0.007706666365265846
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,127,0.007285333548982938
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,127,0.007093333328763644
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,255,0.006826666494210561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,2047,0.012842666357755661
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,255,0.006693333387374878
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,511,0.00874133345981439
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,511,0.008832000195980072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,1023,0.009205333267649015
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,32767,0.08255466818809509
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,2047,0.010949333508809408
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,1023,0.00903466654320558
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,2047,0.010138666878143946
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,4095,0.012789333860079447
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,4095,0.011205332974592844
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,8191,0.01505600040157636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,8191,0.0273333340883255
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,16383,0.04607999821503957
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,16383,0.02924799919128418
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,1,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,3,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,3,0.006517333288987477
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,7,0.007050666958093643
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,1,0.006768000001708667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,7,0.006810666372378667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,15,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,15,0.006624000146985054
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,31,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,63,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,31,0.0069866664707660675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,127,0.0069866664707660675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,127,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,15,0.007029333462317784
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,255,0.0069386667261521024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,31,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,511,0.008762666955590248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,511,0.006981333096822103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,1023,0.009061333412925402
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,1023,0.008767999708652496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,2047,0.010650667051474253
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,2047,0.011578666667143503
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,4095,0.027221334477265675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,4095,0.012928000340859095
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,8191,0.04572799801826477
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,8191,0.027210667729377747
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,16383,0.04587733248869578
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,1,0.008746666833758354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,16383,0.08242666721343994
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,1,0.0069226666043202085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,3,0.00871999996403853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,3,0.006698666761318843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,7,0.00842666688064734
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,7,0.006821333120266597
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,15,0.008453333129485449
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,15,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,31,0.008693333094318708
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,31,0.0069866664707660675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,63,0.0068853336075941724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,127,0.008767999708652496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,127,0.006826666494210561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,255,0.008650666723648706
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,1,0.007114666824539502
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,255,0.006784000123540561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,511,0.008879999940594038
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,511,0.008645333349704742
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,1023,0.01071999967098236
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,1023,0.008799999952316284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,63,0.007098666702707608
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,2047,0.0252960001428922
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,2047,0.01268799975514412
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,255,0.00684799998998642
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,8191,0.04509866734345754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,8191,0.08035199840863545
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,1,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,1,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,3,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,16383,0.08235733211040497
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,3,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,7,0.006858666737874349
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,7,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,15,0.0068853336075941724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,15,0.006602666651209195
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,31,0.0069386667261521024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,31,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,63,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,63,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,127,0.0068693334857622785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,127,0.006704000135262807
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,255,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,511,0.00860799973209699
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,511,0.00895999992887179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,1023,0.008645333349704742
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,1023,0.008863999818762144
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,2047,0.008896000062425932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,63,0.008703999842206636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,2047,0.008943999807039896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,4095,0.008912000184257826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,4095,0.00892800030608972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,8191,0.010981333752473196
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,8191,0.009056000038981438
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,16383,0.013167999684810638
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,16383,0.01310933381319046
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,32767,0.017237332959969837
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,32767,0.017258666455745697
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,65535,0.025349333882331848
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,65535,0.025226667523384094
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,1,0.006746666505932808
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,1,0.006714666883150737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,3,0.006560000280539195
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,3,0.006757333253820737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,7,0.006618666773041089
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,15,0.007600000128149986
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,15,0.006730666384100914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,4095,0.02699200063943863
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,31,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,31,0.006709333509206772
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,63,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,63,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,16383,0.15340266625086466
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,127,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,127,0.00655466690659523
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,255,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,255,0.006741333131988843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,511,0.00860799973209699
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,511,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,1023,0.008581333483258883
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,1023,0.007941333577036858
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,4095,0.008767999708652496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,2047,0.008682666967312494
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,2047,0.008896000062425932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,4095,0.008549333239595095
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,8191,0.00933333362142245
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,8191,0.008693333094318708
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,16383,0.012885333349307379
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,32767,0.017077332983414333
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,65535,0.02142400046189626
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,65535,0.020010666300853092
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,255,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,1,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,1,0.006645333642760913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,3,0.007040000210205714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,3,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,7,0.006768000001708667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,15,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,15,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,31,0.006821333120266597
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,31,0.006698666761318843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,63,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,127,0.0069919998447100324
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,127,0.006810666372378667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,255,0.006751999879876773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,255,0.006741333131988843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,511,0.007290666922926903
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,511,0.008837333569924036
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,7,0.007040000210205714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,1023,0.008682666967312494
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,2047,0.00903466654320558
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,4095,0.04378666480382284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,2047,0.008517333616813024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,4095,0.008933333059151968
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,4095,0.008639999975760778
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,8191,0.009573333586255709
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,8191,0.00891733355820179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,16383,0.014085333794355392
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,16383,0.013056000073750814
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,32767,0.017029333859682083
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,32767,0.015317333241303762
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,16383,0.01303999995191892
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,65535,0.03345600018898646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,65535,0.017210666090250015
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,32767,0.017071999609470367
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,1,0.007071999832987785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,1,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,3,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,3,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,7,0.0069386667261521024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,7,0.0069973332186539965
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,15,0.007040000210205714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,15,0.006837333242098491
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,31,0.00702400008837382
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,31,0.006714666883150737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,7,0.006597333277265231
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,63,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,63,0.00701333334048589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,127,0.00702400008837382
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,127,0.006591999903321266
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,255,0.006725333631038666
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,511,0.00884799969693025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,511,0.008672000219424566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,1023,0.008869333192706108
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,1023,0.008672000219424566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,2047,0.008821333448092142
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,63,0.0074506668994824094
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,2047,0.008778666456540426
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,4095,0.00884799969693025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,4095,0.008805333326260248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,8191,0.010858666151762009
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,8191,0.009002666920423508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,16383,0.013594667116800943
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,16383,0.013072000195582708
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,32767,0.01889066646496455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,32767,0.018170667191346485
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,65535,0.02317333221435547
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,1,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,1,0.006618666773041089
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,3,0.006714666883150737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,3,0.006821333120266597
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,7,0.0068853336075941724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,7,0.00666133314371109
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,15,0.006757333253820737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,15,0.006789333497484525
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,31,0.006671999891599019
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,1023,0.008693333094318708
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,31,0.0068693334857622785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,63,0.006768000001708667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,63,0.006981333096822103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,127,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,255,0.006714666883150737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,255,0.006682666639486949
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,511,0.00867733359336853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,511,0.006805333619316419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,1023,0.00867733359336853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,1023,0.009039999917149544
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,2047,0.008853333070874214
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,4095,0.008645333349704742
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,4095,0.008810666700204214
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,8191,0.01116266722480456
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,8191,0.00871999996403853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,16383,0.014192000031471252
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,16383,0.013125333935022354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,32767,0.01710933322707812
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,65535,0.0335413341720899
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,65535,0.017162666966517765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,1,0.006655999769767125
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,1,0.006789333497484525
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,3,0.006826666494210561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,7,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,255,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,15,0.0068693334857622785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,15,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,31,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,31,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,63,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,65535,0.02110933264096578
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,127,0.0069386667261521024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,127,0.006677333265542984
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,255,0.006671999891599019
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,255,0.006725333631038666
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,511,0.00854399986565113
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,127,0.006704000135262807
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,1023,0.008853333070874214
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,1023,0.008778666456540426
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,2047,0.008613333106040955
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,2047,0.008623999853928884
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,4095,0.008762666955590248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,4095,0.008565333361426989
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,32767,0.015392000476519266
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,8191,0.011674666156371435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,8191,0.010832000523805618
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,16383,0.01292266696691513
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,32767,0.03073599934577942
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,32767,0.016794666647911072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,65535,0.047728002071380615
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,3,0.006720000257094701
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,7,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,65535,0.02937600016593933
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,1,0.007610666876037915
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,1,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,3,0.008672000219424566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,3,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,63,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,7,0.00696000022192796
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,7,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,15,0.007274666801095009
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,15,0.007055999711155891
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,31,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,63,0.007637333124876022
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,511,0.0069919998447100324
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,63,0.007093333328763644
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,127,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,127,0.007029333462317784
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,255,0.006821333120266597
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,2047,0.008693333094318708
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,255,0.006810666372378667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,511,0.008789333204428354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,511,0.00855466661353906
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,1023,0.01089599976936976
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,1023,0.009119999905427298
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,2047,0.01310933381319046
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,2047,0.010949333508809408
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,8191,0.04587733248869578
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,4095,0.027141332626342773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,4095,0.014943999548753103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,8191,0.028698667883872986
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,1,0.00892800030608972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,1,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,7,0.008629333227872849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,7,0.006586666529377301
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,15,0.008581333483258883
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,15,0.007002666592597961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,3,0.008634666601816813
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,3,0.006789333497484525
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,31,0.009018666421373686
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,31,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,63,0.00867733359336853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,63,0.00697066696981589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,127,0.008725333337982496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,127,0.006741333131988843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,255,0.008890666688481966
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,255,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,511,0.009018666421373686
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,511,0.008863999818762144
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,1023,0.011055999745925268
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,1023,0.008789333204428354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,2047,0.027237333357334137
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,2047,0.012709333250919977
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,4095,0.04480533301830292
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,4095,0.027274665733178455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,8191,0.04571199913819631
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,8191,0.0819893330335617
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,3,0.008799999952316284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,3,0.008874666566650072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,1,0.008736000085870424
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,1,0.008613333106040955
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,7,0.008992000172535578
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,15,0.00903466654320558
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,15,0.00879466657837232
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,31,0.008896000062425932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,31,0.00855466661353906
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,63,0.008709333216150602
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,63,0.008816000074148178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,127,0.00961599995692571
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,127,0.008789333204428354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,255,0.009061333412925402
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,255,0.008943999807039896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,511,0.011183999478816986
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,511,0.01102399950226148
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,1023,0.02647999922434489
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,2047,0.04371733466784159
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,31,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,2047,0.0271573339899381
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,4095,0.08049599826335907
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,1,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,4095,0.04582933088143667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,1,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,8191,0.15336533387502035
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,8191,0.08380267024040222
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,3,0.006757333253820737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,7,0.007104000076651573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,7,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,15,0.006773333375652631
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,31,0.008165333420038223
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,31,0.006720000257094701
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,16383,0.015040000279744467
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,63,0.007237333183487256
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,127,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,255,0.007104000076651573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,127,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,255,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,7,0.008650666723648706
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,511,0.008762666955590248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,511,0.00843733362853527
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,1023,0.00879466657837232
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,1023,0.008757333581646284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,2047,0.00892800030608972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,2047,0.008709333216150602
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,4095,0.009114666531483332
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,4095,0.008933333059151968
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,8191,0.01080000028014183
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,8191,0.010117333382368088
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,16383,0.015146666516860327
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,16383,0.014762666076421738
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,32767,0.0189280000825723
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,32767,0.01699200024207433
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,65535,0.033887999753157295
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,65535,0.01930133377512296
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,1,0.00679466687142849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,1,0.006682666639486949
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,3,0.006650666395823161
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,3,0.006720000257094701
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,7,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,7,0.006533333410819371
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,1023,0.013034666577974955
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,15,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,15,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,31,0.006751999879876773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,31,0.0068693334857622785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,127,0.0068693334857622785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,127,0.006618666773041089
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,3,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,255,0.007141333073377609
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,255,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,511,0.006837333242098491
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,511,0.008816000074148178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,1023,0.008933333059151968
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,15,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,1023,0.008837333569924036
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,2047,0.008767999708652496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,2047,0.008597333605090777
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,63,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,4095,0.008874666566650072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,4095,0.00877333308259646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,8191,0.012485332787036896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,8191,0.011018666128317514
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,16383,0.014773332824309668
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,16383,0.012949333836634954
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,32767,0.03109866629044215
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,32767,0.016623999923467636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,65535,0.049327999353408813
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,65535,0.031136001149813335
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,1,0.006618666773041089
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,3,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,3,0.006693333387374878
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,7,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,7,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,15,0.007109333450595538
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,15,0.006650666395823161
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,31,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,63,0.006730666384100914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,127,0.00707733320693175
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,127,0.006634666894872983
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,255,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,255,0.006735999758044879
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,511,0.008709333216150602
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,511,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,63,0.006693333387374878
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,1023,0.008618666479984919
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,2047,0.008997333546479544
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,2047,0.008767999708652496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,4095,0.010960000256697336
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,4095,0.011061333119869232
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,8191,0.014229333649079004
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,8191,0.01268799975514412
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,16383,0.028965334097544353
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,16383,0.015178666760524115
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,32767,0.046869332591692604
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,32767,0.029311999678611755
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,65535,0.08271466692288716
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,65535,0.047728002071380615
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,1,0.007061333085099856
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,1,0.008992000172535578
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,1,0.008661333471536636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,3,0.008586666857202848
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,31,0.006688000013430913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,3,0.00821333316465219
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,63,0.0069973332186539965
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,7,0.008890666688481966
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,7,0.008661333471536636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,15,0.008863999818762144
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,15,0.00795199970404307
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,31,0.008757333581646284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,31,0.007983999947706858
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,63,0.009066666786869368
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,63,0.008650666723648706
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,127,0.008639999975760778
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,1023,0.0085333331177632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,127,0.007370666911204656
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,255,0.009039999917149544
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,255,0.00860799973209699
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,511,0.009759999811649323
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,511,0.009002666920423508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,1023,0.011701333026091257
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,1023,0.01080000028014183
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,2047,0.02700799951950709
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,2047,0.013199999928474426
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,1,0.008805333326260248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,1,0.008858666444818178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,3,0.009119999905427298
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,4095,0.04574400186538696
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,4095,0.027087998886903126
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,7,0.009061333412925402
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,7,0.00879466657837232
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,3,0.009008000294367472
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,15,0.008863999818762144
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,31,0.008661333471536636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,63,0.00903466654320558
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,63,0.008869333192706108
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,63,0.00684799998998642
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,127,0.008896000062425932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,15,0.008650666723648706
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,31,0.008874666566650072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,127,0.008853333070874214
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,255,0.008752000207702318
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,255,0.008890666688481966
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,511,0.010682666053374609
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,511,0.011418666690587997
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,1023,0.027002667387326557
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,4095,0.08258133133252461
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,1023,0.013242666920026144
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,1,0.010832000523805618
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,4095,0.04714666803677877
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,1,0.011653333902359009
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,2047,0.0455626646677653
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,3,0.011018666128317514
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,7,0.010703999549150467
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,3,0.012842666357755661
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,15,0.011450666934251785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,7,0.012752000242471695
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,31,0.01080000028014183
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,63,0.012784000486135483
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,127,0.012655999511480331
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,127,0.01099733387430509
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,255,0.012965332716703415
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,255,0.01089599976936976
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,511,0.027482666075229645
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,511,0.013738666971524557
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,1023,0.045738667249679565
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,1023,0.027162666122118633
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,2047,0.02741333345572154
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,2047,0.0470719983180364
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,15,0.010922666639089584
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,4095,0.15473066767056784
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,1,0.008821333448092142
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,31,0.012448000411192576
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,1,0.010746666540702185
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,3,0.010735999792814255
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,3,0.00914666677514712
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,7,0.010944000134865442
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,7,0.009008000294367472
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,15,0.010741333166758219
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,15,0.008890666688481966
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,2047,0.08052800099054973
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,31,0.010922666639089584
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,31,0.00884799969693025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,63,0.010768000036478043
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,63,0.009072000160813332
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,127,0.01097600037852923
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,4095,0.08236800134181976
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,127,0.008901333436369896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,255,0.010879999647537867
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,511,0.013141332815090815
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,511,0.011146667102972666
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,1023,0.027322667340437572
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,1,0.012351999680201212
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,1023,0.015125333021084467
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,1,0.010922666639089584
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,3,0.012789333860079447
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,3,0.01081066702802976
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,7,0.012421333541472753
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,7,0.010677333921194077
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,15,0.010954666882753372
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,31,0.012794667234023413
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,31,0.010661333799362183
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,63,0.010970667004585266
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,63,0.010794666906197866
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,255,0.008992000172535578
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,127,0.011952000359694162
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,127,0.0107893335322539
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,255,0.012655999511480331
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,255,0.010682666053374609
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,511,0.027130665878454845
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,511,0.013301332791646322
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,1,0.017018667111794155
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,1023,0.045610666275024414
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,1023,0.02771199991305669
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,1,0.01481066644191742
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,15,0.012778667112191519
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,3,0.01699200024207433
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,7,0.016997333616018295
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,63,0.012821332861979803
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,15,0.014757333944241205
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,31,0.016901332885026932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,31,0.014970666418472925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,7,0.015013333410024643
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,15,0.01693333312869072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,63,0.017210666090250015
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,63,0.014981333166360855
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,127,0.016965333372354507
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,127,0.014805333067973455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,255,0.026858667532602947
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,255,0.014794666320085526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,511,0.045093332727750145
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,3,0.014991999914248785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,511,0.027679999669392902
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,1023,0.08089600006739299
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,1023,0.04613333443800608
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,1,0.012885333349307379
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,1,0.015024000157912573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,3,0.013023999830087027
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,3,0.014975999792416891
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,7,0.012800000607967377
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,7,0.014965333044528961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,31,0.014906667172908783
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,31,0.012752000242471695
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,63,0.015002666662136713
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,63,0.012773333738247553
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,15,0.01482133318980535
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,15,0.012768000364303589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,127,0.013034666577974955
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,255,0.01469333345691363
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,255,0.012666666259368261
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,1,0.016837333639462788
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,511,0.01695999999841054
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,1,0.014736000448465347
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,3,0.017045332739750545
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,127,0.014805333067973455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,3,0.014965333044528961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,7,0.014815999815861383
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,7,0.01724799970785777
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,15,0.014896000425020853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,15,0.01720533271630605
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,31,0.017018667111794155
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,31,0.014752000570297241
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,511,0.027621333797772724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,63,0.016837333639462788
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,63,0.014896000425020853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,127,0.0170666662355264
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,127,0.014954666296641031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,255,0.02714666724205017
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,255,0.015072000523408255
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,1,0.027445333699385326
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,511,0.02741333345572154
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,3,0.027434666951497395
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,3,0.023077333966890972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,7,0.02741866558790207
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,7,0.023376000424226124
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,15,0.027066667874654133
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,15,0.023168000082174938
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,31,0.02739199995994568
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,31,0.02293866624434789
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,511,0.045456002155939736
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,63,0.027215999861558277
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,1,0.023333333432674408
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,1,0.006768000001708667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,1,0.006735999758044879
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,63,0.02367999901374181
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,3,0.007189333438873291
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,7,0.006800000245372455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,3,0.006757333253820737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,15,0.007104000076651573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,127,0.02333866556485494
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,15,0.007045333584149678
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,31,0.0068853336075941724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,31,0.007978666573762894
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,255,0.04561600089073181
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,63,0.006831999868154526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,255,0.02881066749493281
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,127,0.007114666824539502
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,255,0.006725333631038666
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,511,0.00874133345981439
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,511,0.08192533254623413
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,511,0.008879999940594038
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,127,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,511,0.04651733239491781
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,255,0.0069866664707660675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,1023,0.008746666833758354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,2047,0.009109333157539368
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,4095,0.008965333302815756
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,8191,0.01309866706530253
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,8191,0.012863999853531519
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,16383,0.014933332800865173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,127,0.029472000896930695
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,1023,0.00902399979531765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,32767,0.03160533308982849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,7,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,2047,0.00873066671192646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,65535,0.033344000577926636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,65535,0.04984533290068308
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,1,0.00697066696981589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,1,0.006634666894872983
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,3,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,3,0.006762666627764702
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,7,0.006778666749596596
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,7,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,15,0.006773333375652631
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,15,0.006671999891599019
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,31,0.006714666883150737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,31,0.006655999769767125
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,63,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,63,0.006682666639486949
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,63,0.006591999903321266
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,127,0.0068853336075941724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,127,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,255,0.00684799998998642
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,255,0.006645333642760913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,511,0.008810666700204214
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,511,0.007157333195209503
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,1023,0.008634666601816813
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,2047,0.008965333302815756
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,2047,0.008863999818762144
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,4095,0.010735999792814255
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,4095,0.01090666651725769
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,8191,0.012725333372751871
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,8191,0.013669333110253016
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,16383,0.029301332930723827
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,16383,0.015125333021084467
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,32767,0.047770669062932335
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,4095,0.009103999783595404
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,32767,0.029477333029111225
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,65535,0.08431999882062276
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,16383,0.015200000256299973
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,1,0.006757333253820737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,1,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,3,0.006784000123540561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,7,0.0069386667261521024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,7,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,15,0.00696000022192796
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,15,0.007376000285148621
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,31,0.006778666749596596
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,31,0.006981333096822103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,32767,0.017557332913080852
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,63,0.007055999711155891
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,63,0.006751999879876773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,127,0.00660800002515316
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,255,0.006757333253820737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,255,0.006741333131988843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,511,0.00879466657837232
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,511,0.007082666580875714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,1023,0.009002666920423508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,2047,0.011002667248249054
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,2047,0.008645333349704742
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,4095,0.012960000584522883
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,8191,0.026949333647886913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,1023,0.008682666967312494
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,8191,0.01309866706530253
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,16383,0.04515733321507772
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,16383,0.02739199995994568
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,32767,0.0813920001188914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,32767,0.04548266530036926
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,65535,0.15357866883277893
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,65535,0.08299200236797333
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,65535,0.047781333327293396
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,3,0.0069919998447100324
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,127,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,1023,0.008714666590094566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,4095,0.010944000134865442
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,1,0.021520001192887623
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,1,0.016821333517630894
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,3,0.022778667509555817
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,7,0.021967999637126923
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,3,0.01704000060757001
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,7,0.01721599946419398
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,15,0.021920000513394673
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,15,0.016917333006858826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,31,0.021962667504946392
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,31,0.01699200024207433
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,63,0.022442666192849476
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,63,0.016800000021855038
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,127,0.02195200075705846
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,127,0.017093333105246227
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,255,0.02731200059254964
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,255,0.017130666722853977
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,1,0.02734400083621343
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,1,0.0230880007147789
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,3,0.027274665733178455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,3,0.02292799949645996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,7,0.027642667293548584
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,7,0.02327466756105423
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,15,0.02752000093460083
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,31,0.027056001126766205
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,15,0.02312533309062322
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,31,0.02334933231274287
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,63,0.023029332359631855
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,63,0.027290667096773785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,127,0.030576000610987347
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,127,0.023189333577950794
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,255,0.04562133550643921
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,1,0.04782933493455251
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,255,0.029152000943819683
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,1,0.03967999915281931
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,3,0.04775466521581014
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,3,0.03951466580231985
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,7,0.04819199939568838
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,7,0.039642666776975
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,15,0.03955733279387156
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,31,0.03953066716591517
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,31,0.048026666045188904
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,63,0.048453330993652344
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,63,0.03959999978542328
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,15,0.04931733508904775
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,127,0.04110399881998698
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,127,0.0516480008761088
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,255,0.08276799817879994
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,255,0.0487306664387385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,1,0.03735466549793879
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,1,0.027087998886903126
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,3,0.027466667195161183
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,3,0.03775466730197271
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,7,0.03787733366092046
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,7,0.027306665976842243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,15,0.027290667096773785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,31,0.027141332626342773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,31,0.03722666700681051
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,15,0.0376800000667572
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,1,0.04831466575463613
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,127,0.03965866565704346
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,127,0.02735999971628189
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,1,0.03969600051641464
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,3,0.047775998711586
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,63,0.03833599885304769
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,63,0.027850667635599773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,3,0.03963200002908707
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,7,0.04854399959246317
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,7,0.0402399996916453
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,15,0.049866666396458946
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,15,0.039306665460268654
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,31,0.04789333542188009
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,63,0.0483893354733785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,63,0.039621333281199135
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,127,0.05165866514046987
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,31,0.03948266555865606
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,1,0.08871466914812724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,1,0.07242133220036824
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,127,0.041573333243529
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,3,0.08814932902654012
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,3,0.07236266632874806
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,7,0.0888213316599528
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,7,0.07256533205509186
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,15,0.08880533774693807
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,31,0.08841600020726521
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,15,0.07226133346557617
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,1,0.007087999954819679
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,3,0.007007999966541926
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,1,0.006720000257094701
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,3,0.007120000198483467
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,7,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,7,0.006837333242098491
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,31,0.07257066667079926
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,15,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,15,0.006784000123540561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,31,0.007055999711155891
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,63,0.006831999868154526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,63,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,127,0.007087999954819679
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,127,0.007226666435599327
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,255,0.00702400008837382
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,255,0.006671999891599019
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,511,0.008992000172535578
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,1023,0.00884799969693025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,1023,0.008863999818762144
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,63,0.08853866656621297
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,2047,0.008997333546479544
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,2047,0.00897066667675972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,4095,0.012960000584522883
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,4095,0.010992000500361124
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,8191,0.013189333180586496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,16383,0.029722665747006733
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,8191,0.014901333798964819
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,16383,0.017071999609470367
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,32767,0.04775999983151754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,32767,0.03046400099992752
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,1,0.006821333120266597
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,1,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,3,0.007007999966541926
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,3,0.006602666651209195
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,7,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,7,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,15,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,15,0.007007999966541926
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,31,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,31,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,63,0.00697066696981589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,63,0.006714666883150737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,31,0.006805333619316419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,127,0.006666666517655055
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,127,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,255,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,63,0.07254933317502339
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,255,0.006805333619316419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,511,0.007040000210205714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,1023,0.009002666920423508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,511,0.008709333216150602
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,1023,0.008858666444818178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,2047,0.00874133345981439
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,4095,0.012661332885424295
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,4095,0.010885333021481832
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,8191,0.013056000073750814
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,127,0.09283733367919922
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,8191,0.027317332724730175
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,16383,0.04565866788228353
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,16383,0.027242665489514668
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,32767,0.04752533137798309
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,32767,0.0827893316745758
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,1,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,3,0.00684799998998642
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,3,0.007087999954819679
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,7,0.007002666592597961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,7,0.0069226666043202085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,15,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,15,0.006730666384100914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,1,0.007098666702707608
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,31,0.007018666714429855
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,63,0.006650666395823161
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,63,0.006666666517655055
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,127,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,127,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,255,0.007237333183487256
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,127,0.07311466832955678
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,255,0.006768000001708667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,511,0.008816000074148178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,511,0.007045333584149678
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,511,0.008618666479984919
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,1023,0.010570666442314783
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,2047,0.010853332777818045
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,2047,0.01101333275437355
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,4095,0.02697066714366277
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,4095,0.01302933320403099
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,8191,0.02722666660944621
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,16383,0.08035733302434285
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,1023,0.008709333216150602
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,2047,0.011087999989589056
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,32767,0.15334399541219076
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,1,0.007007999966541926
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,1,0.00701333334048589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,32767,0.08194666604201
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,3,0.006730666384100914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,3,0.007002666592597961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,7,0.006757333253820737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,7,0.00749333327015241
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,15,0.00701333334048589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,15,0.007045333584149678
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,31,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,31,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,63,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,63,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,127,0.006602666651209195
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,127,0.00660800002515316
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,255,0.0069386667261521024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,31,0.007061333085099856
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,255,0.0069226666043202085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,511,0.008714666590094566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,511,0.007018666714429855
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,1023,0.008799999952316284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,1023,0.008698666468262672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,2047,0.00891733355820179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,8191,0.027077332139015198
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,4095,0.010672000547250112
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,8191,0.012703999876976013
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,16383,0.02737066646416982
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,32767,0.04600533346335093
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,32767,0.08353066444396973
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,1,0.007002666592597961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,1,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,3,0.006677333265542984
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,3,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,7,0.007082666580875714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,16383,0.045642669002215065
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,15,0.0069973332186539965
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,15,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,31,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,31,0.006528000036875407
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,63,0.007002666592597961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,63,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,127,0.007098666702707608
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,127,0.006810666372378667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,255,0.008192000289758047
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,255,0.006789333497484525
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,511,0.008629333227872849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,511,0.0074506668994824094
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,1023,0.009663999701539675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,1023,0.00895999992887179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,2047,0.011658667276302973
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,2047,0.010832000523805618
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,2047,0.01080000028014183
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,4095,0.02743999908367793
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,4095,0.012869333227475485
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,16383,0.04574400186538696
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,4095,0.013056000073750814
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,8191,0.027717334528764088
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,16383,0.08243200182914734
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,8191,0.04549333453178406
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,16383,0.04561600089073181
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,1,0.006602666651209195
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,3,0.00873066671192646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,8191,0.044165333112080894
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,32767,0.08251733581225078
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,32767,0.1563093364238739
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,3,0.0069866664707660675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,7,0.006698666761318843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,7,0.006645333642760913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,15,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,15,0.008767999708652496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,31,0.00855466661353906
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,31,0.0069386667261521024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,63,0.00871999996403853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,63,0.006800000245372455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,127,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,255,0.008245333408315977
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,255,0.006575999781489372
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,511,0.009002666920423508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,511,0.008698666468262672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,1023,0.010853332777818045
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,1023,0.009029333169261614
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,2047,0.026000000536441803
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,2047,0.012853333105643591
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,4095,0.04383466641108195
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,8191,0.04558399816354116
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,8191,0.0804746647675832
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,16383,0.08237333099047343
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,16383,0.1532799998919169
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,1,0.008933333059151968
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,32767,0.2972000042597453
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,1,0.008965333302815756
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,1,0.009018666421373686
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,32767,0.15601600209871927
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,7,0.008799999952316284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,3,0.008858666444818178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,7,0.008885333314538002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,7,0.008879999940594038
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,15,0.008736000085870424
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,15,0.00877333308259646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,31,0.008965333302815756
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,31,0.00873066671192646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,127,0.00877333308259646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,63,0.00890666681031386
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,63,0.008672000219424566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,127,0.00895999992887179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,255,0.00874133345981439
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,255,0.008623999853928884
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,511,0.011663999408483505
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,511,0.010618666807810465
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,1023,0.025914666553338368
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,1023,0.012815999488035837
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,2047,0.043749332427978516
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,4095,0.04590400060017904
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,2047,0.02714666724205017
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,4095,0.08037333190441132
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,8191,0.1536746621131897
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,8191,0.08265600105126698
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,1,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,1,0.006640000268816948
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,16383,0.15668800473213196
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,16383,0.29918400446573895
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,3,0.006693333387374878
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,3,0.008997333546479544
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,32767,0.3023146589597066
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,3,0.006800000245372455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,7,0.006735999758044879
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,15,0.006570666407545407
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,32767,0.5901813507080078
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,31,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,31,0.0068693334857622785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,15,0.006677333265542984
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,63,0.006400000303983688
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,127,0.007029333462317784
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,127,0.006810666372378667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,255,0.007018666714429855
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,63,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,511,0.008912000184257826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,511,0.007130666946371396
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,1023,0.00902399979531765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,1023,0.008789333204428354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,2047,0.0069226666043202085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,4095,0.008746666833758354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,4095,0.008693333094318708
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,8191,0.009237333511312803
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,8191,0.009018666421373686
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,16383,0.012842666357755661
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,16383,0.012991999586423239
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,32767,0.01711999997496605
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,32767,0.016762666404247284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,65535,0.025231999655564625
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,65535,0.023413332800070446
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,131071,0.02942933390537898
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,131071,0.02737066646416982
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,1,0.0069973332186539965
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,1,0.006757333253820737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,3,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,3,0.006602666651209195
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,7,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,7,0.006858666737874349
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,15,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,15,0.00666133314371109
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,31,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,31,0.00660800002515316
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,63,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,63,0.00666133314371109
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,127,0.006778666749596596
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,127,0.006666666517655055
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,255,0.006762666627764702
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,255,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,511,0.008842666943868002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,511,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,7,0.007120000198483467
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,1023,0.008879999940594038
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,1023,0.008789333204428354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,2047,0.008965333302815756
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,2047,0.00895999992887179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,4095,0.008986666798591614
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,4095,0.009077333534757296
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,8191,0.009178666397929192
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,8191,0.008709333216150602
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,16383,0.01313599944114685
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,16383,0.012997332960367203
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,32767,0.017103999853134155
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,32767,0.017093333105246227
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,127,0.008879999940594038
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,65535,0.021359999974568684
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,131071,0.039520000418027244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,131071,0.021253332495689392
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,1,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,1,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,3,0.0069866664707660675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,3,0.00684799998998642
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,7,0.006784000123540561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,7,0.00655466690659523
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,4095,0.026426665484905243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,15,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,15,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,31,0.0069919998447100324
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,31,0.006618666773041089
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,63,0.006671999891599019
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,63,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,127,0.006677333265542984
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,127,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,255,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,255,0.006698666761318843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,511,0.008858666444818178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,1023,0.007040000210205714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,2047,0.008938666433095932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,2047,0.008832000195980072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,4095,0.008650666723648706
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,8191,0.010015999898314476
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,8191,0.008714666590094566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,16383,0.01333333303531011
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,16383,0.012800000607967377
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,32767,0.017055999487638474
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,32767,0.015157333264748255
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,65535,0.03356799980004629
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,65535,0.017103999853134155
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,255,0.006544000158707301
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,131071,0.05146666864554087
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,131071,0.03461333364248276
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,2047,0.008933333059151968
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,1,0.006730666384100914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,1,0.0069386667261521024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,65535,0.01956266661485036
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,3,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,7,0.006629333520929019
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,7,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,15,0.007125333572427432
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,15,0.006831999868154526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,31,0.006709333509206772
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,31,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,63,0.0069973332186539965
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,63,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,127,0.006655999769767125
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,255,0.007002666592597961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,255,0.006800000245372455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,511,0.008512000242869059
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,511,0.0069919998447100324
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,1023,0.008992000172535578
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,1023,0.008703999842206636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,1023,0.008650666723648706
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,2047,0.008736000085870424
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,4095,0.00891733355820179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,2047,0.00860799973209699
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,4095,0.008821333448092142
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,4095,0.00898133342464765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,8191,0.012762666990359625
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,8191,0.010832000523805618
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,16383,0.01302933320403099
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,16383,0.015141333142916361
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,32767,0.03959999978542328
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,65535,0.03126933425664902
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,65535,0.047983999053637184
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,131071,0.049728001157442726
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,1,0.006981333096822103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,1,0.00666133314371109
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,3,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,3,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,7,0.006688000013430913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,7,0.006560000280539195
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,15,0.0068693334857622785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,15,0.0069386667261521024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,3,0.006698666761318843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,31,0.006778666749596596
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,63,0.006698666761318843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,63,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,127,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,127,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,255,0.006624000146985054
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,255,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,127,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,511,0.008816000074148178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,511,0.007231999809543292
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,1023,0.008618666479984919
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,1023,0.008949333180983862
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,2047,0.008656000097592672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,511,0.007349333415428798
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,2047,0.008592000231146812
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,4095,0.008976000050703684
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,4095,0.008757333581646284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,8191,0.010944000134865442
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,8191,0.009114666531483332
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,16383,0.012725333372751871
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,32767,0.017242666333913803
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,32767,0.01693333312869072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,65535,0.02117866774400075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,65535,0.01924266666173935
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,131071,0.035973332822322845
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,131071,0.020960000654061634
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,1,0.007061333085099856
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,1,0.00660800002515316
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,3,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,3,0.006666666517655055
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,7,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,7,0.006624000146985054
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,15,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,131071,0.08509866396586101
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,15,0.006645333642760913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,31,0.0069386667261521024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,31,0.006682666639486949
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,63,0.0068693334857622785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,127,0.006746666505932808
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,127,0.0068693334857622785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,255,0.006821333120266597
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,255,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,511,0.0086666668454806
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,511,0.007130666946371396
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,1023,0.008746666833758354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,1023,0.00874133345981439
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,2047,0.008639999975760778
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,2047,0.008602666358153025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,4095,0.008581333483258883
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,8191,0.010533332824707031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,8191,0.008757333581646284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,16383,0.013877333452304205
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,16383,0.013258667041858038
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,32767,0.014874666929244995
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,32767,0.01691199963291486
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,65535,0.03307733436425527
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,65535,0.01718933383623759
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,131071,0.051642666260401406
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,131071,0.034501334031422935
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,1,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,1,0.006613333399097125
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,3,0.00707733320693175
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,3,0.006821333120266597
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,16383,0.01310933381319046
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,7,0.006650666395823161
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,7,0.006789333497484525
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,15,0.00707733320693175
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,15,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,31,0.006714666883150737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,31,0.006693333387374878
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,63,0.007007999966541926
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,63,0.006810666372378667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,127,0.007146666447321574
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,127,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,255,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,32767,0.01639466608564059
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,255,0.0068853336075941724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,511,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,1023,0.00902399979531765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,1023,0.008757333581646284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,2047,0.008826666822036108
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,2047,0.008752000207702318
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,4095,0.009077333534757296
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,4095,0.008703999842206636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,8191,0.012784000486135483
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,8191,0.010922666639089584
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,63,0.006714666883150737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,16383,0.015157333264748255
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,16383,0.013056000073750814
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,32767,0.030378667016824085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,32767,0.015360000232855478
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,65535,0.031141333281993866
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,65535,0.04853333532810211
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,131071,0.08502933382987976
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,131071,0.049216002225875854
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,1,0.006730666384100914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,1,0.006698666761318843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,3,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,4095,0.008954666554927826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,3,0.006826666494210561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,7,0.006730666384100914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,7,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,15,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,31,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,15,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,31,0.006565333033601443
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,63,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,63,0.006655999769767125
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,127,0.0069973332186539965
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,127,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,255,0.006981333096822103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,255,0.0068693334857622785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,511,0.008629333227872849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,511,0.00696000022192796
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,1023,0.008645333349704742
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,2047,0.008949333180983862
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,2047,0.0086666668454806
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,4095,0.01055466632048289
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,4095,0.010703999549150467
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,8191,0.014773332824309668
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,16383,0.038719999293486275
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,16383,0.014736000448465347
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,32767,0.05750933289527893
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,32767,0.02900800108909607
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,65535,0.08807466427485149
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,131071,0.1554026703039805
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,131071,0.08423999945322673
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,31,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,1,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,1,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,3,0.0068693334857622785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,3,0.006666666517655055
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,7,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,7,0.00684799998998642
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,15,0.007002666592597961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,15,0.00684799998998642
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,31,0.0068853336075941724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,1023,0.008757333581646284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,31,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,63,0.0069226666043202085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,63,0.00874133345981439
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,127,0.007541333635648091
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,127,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,255,0.00701333334048589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,255,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,511,0.008709333216150602
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,511,0.0069919998447100324
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,1023,0.009290666629870733
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,1023,0.008703999842206636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,65535,0.04773333172003428
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,2047,0.01089599976936976
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,8191,0.036517334481080375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,4095,0.027024000883102417
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,16383,0.08292800188064575
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,16383,0.04621866842110952
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,1,0.00873066671192646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,1,0.007061333085099856
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,3,0.008869333192706108
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,3,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,7,0.008047999814152718
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,7,0.007104000076651573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,15,0.008538666491707167
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,15,0.0068693334857622785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,31,0.008874666566650072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,31,0.00697066696981589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,8191,0.012885333349307379
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,63,0.00878399983048439
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,63,0.007141333073377609
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,127,0.00855466661353906
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,127,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,511,0.008661333471536636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,255,0.00860799973209699
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,255,0.007146666447321574
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,511,0.009039999917149544
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,511,0.008752000207702318
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,2047,0.010826667149861654
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,1023,0.0107893335322539
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,8191,0.045738667249679565
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,4095,0.012736000120639801
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,1023,0.008997333546479544
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,4095,0.045594667394955955
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,2047,0.012789333860079447
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,2047,0.027493332823117573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,8191,0.04577066500981649
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,4095,0.027056001126766205
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,1,0.008997333546479544
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,8191,0.08205333352088928
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,7,0.00895999992887179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,7,0.008853333070874214
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,3,0.00890666681031386
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,15,0.009061333412925402
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,1,0.00879466657837232
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,15,0.008656000097592672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,16383,0.08227733274300893
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,3,0.00891733355820179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,16383,0.15610667069753012
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,31,0.008890666688481966
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,63,0.008816000074148178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,63,0.008885333314538002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,127,0.009077333534757296
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,255,0.00897066667675972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,255,0.008890666688481966
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,127,0.009056000038981438
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,511,0.012784000486135483
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,511,0.010821333775917688
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,1023,0.026389333109060924
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,2047,0.043653334180514015
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,1023,0.013130666067202887
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,4095,0.08100266754627228
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,4095,0.04539200166861216
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,8191,0.08292800188064575
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,8191,0.15335999925931296
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,1,0.012421333541472753
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,16383,0.15744533141454062
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,1,0.011002667248249054
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,16383,0.30010666449864704
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,3,0.012362666428089142
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,3,0.010784000158309937
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,7,0.01251199965675672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,31,0.009039999917149544
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,15,0.010687999427318573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,31,0.011978667229413986
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,15,0.01312000056107839
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,31,0.010757333288590113
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,63,0.010703999549150467
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,127,0.012757333616415659
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,127,0.010981333752473196
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,255,0.012709333250919977
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,255,0.010768000036478043
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,2047,0.027221334477265675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,511,0.026394667724768322
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,511,0.014208000153303146
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,1023,0.04480533301830292
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,1023,0.027327999472618103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,2047,0.08111466467380524
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,2047,0.04568000137805939
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,8191,0.30054400364557904
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,8191,0.15635200341542563
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,4095,0.08225599924723308
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,4095,0.15441600481669107
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,1,0.006784000123540561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,1,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,3,0.007040000210205714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,7,0.010949333508809408
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,16383,0.6045973300933838
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,3,0.006591999903321266
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,7,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,7,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,15,0.006640000268816948
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,15,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,31,0.00696000022192796
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,16383,0.3029866615931193
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,31,0.006704000135262807
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,63,0.0068853336075941724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,127,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,255,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,511,0.007098666702707608
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,1023,0.008954666554927826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,63,0.012794667234023413
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,1023,0.00867733359336853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,2047,0.008986666798591614
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,255,0.006784000123540561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,2047,0.00891733355820179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,4095,0.008736000085870424
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,4095,0.008799999952316284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,8191,0.010944000134865442
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,8191,0.008933333059151968
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,16383,0.014101333916187286
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,16383,0.012757333616415659
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,32767,0.017237332959969837
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,32767,0.015253332753976187
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,65535,0.03205333401759466
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,65535,0.017077332983414333
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,131071,0.05029866596062978
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,131071,0.03277866790692011
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,1,0.007098666702707608
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,1,0.006826666494210561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,3,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,3,0.006634666894872983
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,7,0.007007999966541926
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,7,0.007242666557431221
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,15,0.006768000001708667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,15,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,31,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,63,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,63,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,127,0.006805333619316419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,127,0.006837333242098491
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,255,0.006768000001708667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,255,0.006735999758044879
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,511,0.008890666688481966
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,511,0.007146666447321574
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,1023,0.008885333314538002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,2047,0.008687999720374743
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,2047,0.008832000195980072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,4095,0.009002666920423508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,4095,0.008789333204428354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,8191,0.01190399999419848
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,8191,0.010832000523805618
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,16383,0.01526933287580808
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,16383,0.013445333888133367
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,32767,0.03109866629044215
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,32767,0.015370666980743408
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,65535,0.04891733328501383
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,63,0.0069386667261521024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,127,0.006831999868154526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,65535,0.030005333324273426
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,511,0.00871999996403853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,131071,0.08613866567611694
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,131071,0.050154666105906166
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,3,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,3,0.007007999966541926
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,7,0.007104000076651573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,7,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,15,0.007061333085099856
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,15,0.006544000158707301
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,31,0.006698666761318843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,31,0.007055999711155891
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,63,0.006613333399097125
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,63,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,127,0.006688000013430913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,127,0.006821333120266597
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,255,0.006655999769767125
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,31,0.006693333387374878
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,511,0.008538666491707167
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,511,0.007045333584149678
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,1023,0.008752000207702318
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,1023,0.008693333094318708
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,2047,0.008757333581646284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,4095,0.010879999647537867
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,4095,0.00973866693675518
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,8191,0.014192000031471252
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,8191,0.01246400053302447
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,16383,0.02898666759332021
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,32767,0.047093331813812256
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,32767,0.02903466671705246
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,1,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,65535,0.08239999910195668
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,65535,0.0481333335240682
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,131071,0.08594666918118794
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,131071,0.15458133816719055
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,1,0.00666133314371109
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,255,0.00701333334048589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,3,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,3,0.006682666639486949
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,7,0.007055999711155891
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,7,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,2047,0.008687999720374743
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,1023,0.008885333314538002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,15,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,15,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,31,0.006624000146985054
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,63,0.006826666494210561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,127,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,127,0.0069973332186539965
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,255,0.007114666824539502
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,255,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,511,0.008687999720374743
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,511,0.007162666569153468
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,1023,0.008922666932145754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,2047,0.010911999891201654
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,1,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,4095,0.012975999464591345
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,4095,0.010960000256697336
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,8191,0.01301866645614306
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,8191,0.03044266750415166
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,16383,0.02731200059254964
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,16383,0.056101332108179726
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,1,0.007162666569153468
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,32767,0.04502933224042257
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,32767,0.0809386670589447
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,65535,0.15217066804567972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,31,0.00684799998998642
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,63,0.006768000001708667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,131071,0.2982986569404602
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,16383,0.014842666685581207
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,131071,0.1552853286266327
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,1,0.007007999966541926
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,1023,0.008709333216150602
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,3,0.008613333106040955
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,3,0.006773333375652631
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,2047,0.00877333308259646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,7,0.008965333302815756
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,7,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,15,0.008698666468262672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,15,0.007055999711155891
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,31,0.00878399983048439
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,31,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,63,0.008896000062425932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,63,0.006714666883150737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,127,0.008767999708652496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,127,0.008266666904091835
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,255,0.00873066671192646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,255,0.00696000022192796
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,511,0.008821333448092142
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,511,0.008901333436369896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,1023,0.010773333410422007
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,1023,0.009205333267649015
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,65535,0.08063466846942902
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,2047,0.026373334228992462
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,2047,0.013013333082199097
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,1,0.008736000085870424
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,4095,0.045312002301216125
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,8191,0.04612799982229868
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,4095,0.02716800073782603
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,1,0.008938666433095932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,1,0.008634666601816813
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,3,0.009125333279371262
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,7,0.00903466654320558
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,15,0.008879999940594038
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,15,0.00874133345981439
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,31,0.009253333633144697
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,3,0.008896000062425932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,31,0.008602666358153025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,7,0.008810666700204214
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,63,0.008954666554927826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,63,0.008639999975760778
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,127,0.00902399979531765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,127,0.00890666681031386
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,255,0.009029333169261614
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,255,0.008709333216150602
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,511,0.012319999436537424
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,1023,0.02699733277161916
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,1023,0.012970666090647379
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,8191,0.0836853285630544
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,2047,0.04574400186538696
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,4095,0.08329600095748901
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,2047,0.027424000203609467
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,4095,0.04650133351484934
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,1,0.010826667149861654
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,8191,0.08351999521255493
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,8191,0.15643733739852905
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,3,0.010746666540702185
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,7,0.012752000242471695
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,7,0.010746666540702185
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,1,0.012063999970753988
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,3,0.012522666404644648
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,15,0.012661332885424295
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,15,0.01101333275437355
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,31,0.010784000158309937
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,31,0.012026666353146235
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,63,0.012938667088747025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,63,0.01062400018175443
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,127,0.012527999778588613
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,511,0.010618666807810465
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,127,0.010768000036478043
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,255,0.01301866645614306
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,255,0.010879999647537867
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,511,0.02643200010061264
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,511,0.013487999637921652
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,2047,0.046165332198143005
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,2047,0.08171199758847554
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,4095,0.08348799745241801
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,4095,0.1548853317896525
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,1023,0.027114666998386383
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,1023,0.044719999035199486
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,1,0.01716800034046173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,1,0.014837333311637243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,3,0.014896000425020853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,8191,0.156741331020991
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,15,0.017130666722853977
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,8191,0.3052320082982381
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,15,0.014901333798964819
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,31,0.017071999609470367
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,31,0.014698666830857595
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,63,0.017050666113694508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,7,0.014789332946141561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,7,0.016965333372354507
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,127,0.016751999656359356
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,63,0.015114666273196539
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,127,0.015109332899252573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,255,0.025311999022960663
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,255,0.015168000012636185
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,1023,0.04533333579699198
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,1023,0.08022400240103404
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,2047,0.15228266517321268
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,3,0.01691199963291486
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,511,0.02736533433198929
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,511,0.04394666850566864
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,2047,0.08243733147780101
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,4095,0.1563040018081665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,4095,0.2957386573155721
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,1,0.008832000195980072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,8191,0.3033386667569478
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,8191,0.5825386842091879
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,3,0.008736000085870424
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,7,0.008874666566650072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,7,0.008938666433095932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,15,0.008853333070874214
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,31,0.008799999952316284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,31,0.00902399979531765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,63,0.00897066667675972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,63,0.009002666920423508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,127,0.009088000282645226
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,127,0.008639999975760778
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,255,0.008842666943868002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,255,0.009008000294367472
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,511,0.011365332951148352
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,511,0.010954666882753372
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,1023,0.02733866622050603
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,1,0.008682666967312494
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,15,0.009178666397929192
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,1023,0.013023999830087027
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,2047,0.045663997530937195
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,2047,0.02717333287000656
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,1,0.010773333410422007
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,1,0.012863999853531519
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,3,0.011786667009194693
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,3,0.010933333386977514
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,15,0.011621333658695221
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,15,0.010837333897749582
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,31,0.012159999459981918
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,7,0.012005332857370377
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,7,0.010981333752473196
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,31,0.010911999891201654
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,63,0.012085333466529846
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,3,0.008821333448092142
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,63,0.010768000036478043
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,127,0.012778667112191519
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,255,0.011717333147923151
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,255,0.010928000013033548
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,511,0.027317332724730175
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,511,0.012949333836634954
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,1,0.01714666684468587
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,1023,0.02812800059715907
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,1023,0.04548266530036926
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,1,0.01524266724785169
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,3,0.017173333714405697
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,3,0.01471466695268949
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,2047,0.08264000217119853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,127,0.01073066641887029
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,15,0.01724799970785777
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,15,0.014815999815861383
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,7,0.01695466662446658
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,7,0.014805333067973455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,31,0.01717866708834966
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,63,0.017103999853134155
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,63,0.01482133318980535
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,127,0.016906666258970898
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,127,0.014831999937693277
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,255,0.026416001220544178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,255,0.015082667271296183
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,2047,0.04610666632652283
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,31,0.014885333677132925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,511,0.02734400083621343
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,2047,0.08345599969228108
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,2047,0.15264532963434854
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,1,0.023445333043734234
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,1023,0.08046933511892955
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,1023,0.04595200220743815
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,1,0.027317332724730175
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,3,0.0232640008131663
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,7,0.02739199995994568
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,3,0.027317332724730175
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,7,0.02310933421055476
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,511,0.044362664222717285
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,31,0.02739199995994568
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,31,0.02306666721900304
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,63,0.027130665878454845
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,63,0.023120000958442688
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,127,0.035887998839219414
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,127,0.023344000180562336
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,15,0.023354666928450268
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,15,0.027301333844661713
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,255,0.04504533112049103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,255,0.02792000025510788
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,1023,0.1520639955997467
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,1023,0.08063999811808269
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,2047,0.15242133537928262
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,511,0.08064533273379008
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,511,0.04577599962552389
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,2047,0.2951253255208333
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,3,0.013077333569526672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,1,0.012709333250919977
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,1,0.01090666651725769
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,3,0.01080000028014183
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,31,0.012762666990359625
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,15,0.01108266661564509
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,7,0.010714666297038397
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,31,0.010826667149861654
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,63,0.012789333860079447
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,15,0.013013333082199097
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,63,0.010805333654085795
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,127,0.012858666479587555
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,127,0.01071999967098236
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,255,0.012485332787036896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,7,0.012896000097195307
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,255,0.010768000036478043
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,511,0.027717334528764088
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,511,0.013114667187134424
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,1023,0.04577599962552389
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,1,0.01695999999841054
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,1023,0.027930667002995808
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,1,0.014864000181357065
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,3,0.01716800034046173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,3,0.015072000523408255
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,7,0.017210666090250015
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,7,0.01522133375207583
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,15,0.016927999754746754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,15,0.014789332946141561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,31,0.014736000448465347
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,63,0.016895999511082966
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,31,0.01714666684468587
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,63,0.015072000523408255
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,127,0.017173333714405697
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,127,0.014922666052977243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,255,0.027087998886903126
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,255,0.014730667074521383
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,511,0.02788266787926356
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,1,0.027322667340437572
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,1,0.023082666099071503
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,1023,0.08189866443475087
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,1023,0.04593066871166229
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,3,0.027466667195161183
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,3,0.022997332115968067
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,7,0.027066667874654133
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,15,0.027098665634791057
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,7,0.023157333334287006
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,511,0.04560533165931702
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,31,0.023039999107519787
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,63,0.02700799951950709
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,63,0.022965334355831146
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,255,0.04533866544564565
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,127,0.02942933390537898
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,127,0.02329600105683009
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,31,0.027488000690937042
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,255,0.02916266769170761
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,511,0.08061333497365315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,1,0.04781333108743032
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,511,0.04608533283074697
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,1,0.03958933303753535
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,15,0.02333866556485494
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,3,0.04770133395989736
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,7,0.04772266745567322
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,1023,0.1530026694138845
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,15,0.04790399968624115
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,15,0.039690665900707245
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,1023,0.08344533046086629
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,3,0.03938133269548416
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,31,0.04837866624196371
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,7,0.039493332306543984
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,31,0.03966933240493139
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,63,0.0396373321612676
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,127,0.04976533353328705
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,255,0.08567999800046285
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,255,0.047839999198913574
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,1023,0.2956053415934245
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,3,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,3,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,7,0.007071999832987785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,7,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,15,0.006831999868154526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,1023,0.15295466780662537
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,31,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,31,0.006720000257094701
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,127,0.04121066629886627
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,511,0.15221866965293884
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,63,0.00679466687142849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,63,0.049365331729253135
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,511,0.0820000022649765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,1,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,127,0.006789333497484525
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,1,0.006746666505932808
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,255,0.006751999879876773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,511,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,1023,0.008725333337982496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,1023,0.00860799973209699
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,2047,0.008816000074148178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,2047,0.009253333633144697
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,4095,0.008986666798591614
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,4095,0.008890666688481966
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,8191,0.011525332927703857
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,8191,0.011141333729028702
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,16383,0.013088000317414602
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,63,0.00696000022192796
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,32767,0.029696000119050343
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,32767,0.015178666760524115
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,65535,0.031317333380381264
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,131071,0.08583466211954753
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,131071,0.05022933085759481
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,15,0.006768000001708667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,1,0.006688000013430913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,3,0.007114666824539502
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,3,0.006618666773041089
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,7,0.007055999711155891
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,7,0.006597333277265231
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,15,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,15,0.006645333642760913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,31,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,31,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,63,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,63,0.006634666894872983
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,511,0.008933333059151968
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,127,0.006624000146985054
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,255,0.006735999758044879
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,255,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,511,0.008714666590094566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,511,0.00679466687142849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,255,0.006762666627764702
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,16383,0.014938666174809137
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,1023,0.008912000184257826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,1023,0.008714666590094566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,2047,0.009002666920423508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,4095,0.011306667079528173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,65535,0.04800533254941305
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,4095,0.01099733387430509
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,8191,0.013530666629473368
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,8191,0.012639999389648438
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,16383,0.029274667302767437
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,1,0.006666666517655055
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,16383,0.014864000181357065
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,32767,0.04753066599369049
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,32767,0.029658667743206024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,65535,0.04818666477998098
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,1,0.007002666592597961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,131071,0.15777599811553955
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,1,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,3,0.006602666651209195
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,3,0.006581333155433337
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,7,0.006677333265542984
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,7,0.007029333462317784
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,127,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,15,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,15,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,127,0.006751999879876773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,31,0.006773333375652631
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,31,0.006751999879876773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,63,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,63,0.006730666384100914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,127,0.006800000245372455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,127,0.0074346667776505155
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,255,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,255,0.006805333619316419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,511,0.008687999720374743
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,511,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,1023,0.009002666920423508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,1023,0.008858666444818178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,2047,0.01071999967098236
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,2047,0.008869333192706108
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,4095,0.012810666114091873
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,4095,0.010650667051474253
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,8191,0.013002666334311167
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,8191,0.026634665826956432
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,16383,0.027679999669392902
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,16383,0.04576000074545542
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,32767,0.08086933195590973
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,32767,0.04566933214664459
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,65535,0.15421332915623984
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,65535,0.08447999755541484
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,65535,0.0843999981880188
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,1,0.006757333253820737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,3,0.0069386667261521024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,3,0.006650666395823161
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,131071,0.29838399092356366
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,7,0.0068693334857622785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,7,0.0069226666043202085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,15,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,15,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,31,0.007082666580875714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,63,0.00701333334048589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,63,0.006655999769767125
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,127,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,127,0.006778666749596596
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,255,0.006800000245372455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,255,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,511,0.00890666681031386
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,511,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,1023,0.009103999783595404
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,1023,0.008687999720374743
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,2047,0.01209066684047381
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,2047,0.008869333192706108
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,2047,0.010672000547250112
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,4095,0.03623466690381368
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,4095,0.012965332716703415
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,8191,0.02736533433198929
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,8191,0.055045331517855324
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,16383,0.08229333162307739
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,16383,0.04507199923197428
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,32767,0.08145066599051158
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,32767,0.15195199847221375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,131071,0.08780266841252644
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,65535,0.2953280011812846
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,65535,0.15357866883277893
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,31,0.006826666494210561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,131071,0.2977919975916545
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,131071,0.5846506754557291
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,1,0.0068853336075941724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,131071,0.15599466363588968
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,1,0.017263999829689663
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,1,0.014975999792416891
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,3,0.017103999853134155
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,3,0.015103999525308609
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,7,0.01699200024207433
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,7,0.014938666174809137
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,15,0.016927999754746754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,15,0.014848000059525171
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,31,0.014858666807413101
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,31,0.017071999609470367
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,63,0.017317333569129307
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,127,0.017024000485738117
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,63,0.014943999548753103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,127,0.014896000425020853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,255,0.027045334378878277
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,255,0.014949332922697067
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,511,0.04590400060017904
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,1,0.027221334477265675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,1,0.023221333821614582
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,511,0.02810666710138321
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,3,0.027429332335789997
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,3,0.023141334454218548
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,7,0.02703999976317088
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,7,0.023039999107519787
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,15,0.0230880007147789
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,15,0.02735999971628189
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,31,0.027290667096773785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,63,0.027434666951497395
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,63,0.02309333284695943
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,31,0.023354666928450268
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,127,0.029722665747006733
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,127,0.02327999969323476
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,255,0.04598399996757507
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,1,0.04764266808827718
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,1,0.039701332648595176
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,255,0.029418667157491047
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,511,0.0469813346862793
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,511,0.08264000217119853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,3,0.047653332352638245
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,3,0.03969600051641464
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,7,0.039408000806967415
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,7,0.04796266555786133
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,15,0.04805333415667216
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,63,0.04852266609668732
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,15,0.03965333352486292
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,63,0.039520000418027244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,31,0.04786133269468943
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,31,0.039733332892258964
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,127,0.04983466863632202
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,127,0.04020266731580099
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,255,0.04773333172003428
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,255,0.08217599987983704
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,1,0.08839999636014302
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,1,0.07235200206438701
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,511,0.08282133440176646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,511,0.15425599614779154
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,3,0.08769599596659343
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,7,0.08809600273768108
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,7,0.07249066730340321
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,15,0.08891200025876363
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,15,0.0724373310804367
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,3,0.0726453314224879
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,63,0.08886399865150452
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,63,0.07223999996980031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,127,0.07216533521811168
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,127,0.09131200114885966
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,31,0.08896000186602275
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,31,0.07252266506354015
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,255,0.1535360018412272
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,255,0.08806932965914409
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,1,0.027141332626342773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,511,0.29598400990168255
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,1,0.023290666441122692
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,511,0.15396266182263693
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,3,0.027349332968393963
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,3,0.02314666658639908
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,15,0.02735466758410136
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,7,0.02719466636578242
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,7,0.023381332556406658
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,31,0.0276053324341774
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,63,0.02752533306678136
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,63,0.023376000424226124
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,15,0.023258666197458904
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,31,0.023247999449570973
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,127,0.030879999200503033
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,127,0.023317334552605946
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,1,0.04892266790072123
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,255,0.04651733239491781
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,1,0.039690665900707245
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,255,0.029322666426499683
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,3,0.04854933420817057
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,3,0.039477333426475525
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,7,0.04886400202910105
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,15,0.047968000173568726
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,7,0.0395359992980957
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,15,0.03965333352486292
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,31,0.04804266492525736
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,31,0.03968533376852671
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,63,0.03939199944337209
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,63,0.047824000318845115
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,127,0.05161599814891815
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,1,0.08775466680526733
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,255,0.0846560001373291
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,255,0.05005866785844167
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,1,0.07322133580843608
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,127,0.04144533226887385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,3,0.08778666456540425
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,3,0.07242133220036824
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,7,0.0721973329782486
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,31,0.08822932839393616
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,15,0.07254933317502339
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,15,0.08832533160845439
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,7,0.08885332942008972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,63,0.0881813367207845
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,31,0.0724533349275589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,63,0.07218666871388753
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,127,0.09277333815892537
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,127,0.07249600191911061
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,1,0.1680799921353658
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,1,0.13646399974822998
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,255,0.15573333700497946
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,255,0.08852799733479817
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,3,0.1676959991455078
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,3,0.13607999682426453
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,15,0.16765866676966348
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,15,0.13576533397038779
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,7,0.1678933302561442
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,7,0.13595733046531677
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,63,0.16779200236002603
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,63,0.13659733533859253
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,1,0.0069386667261521024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,31,0.1678453286488851
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,3,0.006714666883150737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,7,0.007002666592597961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,7,0.006666666517655055
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,15,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,31,0.13645866513252258
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,15,0.007002666592597961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,31,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,63,0.007029333462317784
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,63,0.006634666894872983
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,3,0.006858666737874349
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,127,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,127,0.007055999711155891
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,255,0.007018666714429855
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,511,0.008703999842206636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,255,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,31,0.007045333584149678
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,1,0.0069866664707660675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,1023,0.008709333216150602
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,1023,0.008869333192706108
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,2047,0.008698666468262672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,4095,0.010853332777818045
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,4095,0.010677333921194077
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,8191,0.015103999525308609
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,8191,0.012991999586423239
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,16383,0.029258665939172108
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,16383,0.01523200049996376
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,32767,0.04764799773693085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,127,0.1727466583251953
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,32767,0.029391999046007793
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,65535,0.047770669062932335
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,127,0.13781866431236267
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,1,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,3,0.007045333584149678
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,7,0.006768000001708667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,7,0.006565333033601443
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,511,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,15,0.007071999832987785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,15,0.006698666761318843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,2047,0.008879999940594038
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,31,0.007040000210205714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,31,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,63,0.006506666541099548
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,127,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,127,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,255,0.006778666749596596
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,255,0.006720000257094701
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,1,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,511,0.00877333308259646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,511,0.00707733320693175
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,1023,0.008943999807039896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,1023,0.008709333216150602
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,2047,0.010656000425418219
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,2047,0.008709333216150602
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,3,0.006741333131988843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,4095,0.012703999876976013
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,65535,0.08462933699289958
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,4095,0.010847999403874079
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,8191,0.013023999830087027
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,8191,0.027061333258946735
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,16383,0.04573333263397217
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,32767,0.08275199929873149
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,32767,0.04603200157483419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,65535,0.15582399566968283
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,65535,0.08442667126655579
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,1,0.006778666749596596
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,1,0.006682666639486949
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,3,0.006666666517655055
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,3,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,7,0.006778666749596596
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,15,0.0069919998447100324
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,15,0.006741333131988843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,31,0.006800000245372455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,31,0.00697066696981589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,63,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,63,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,63,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,127,0.006831999868154526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,127,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,255,0.169429341952006
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,255,0.006725333631038666
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,511,0.008714666590094566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,255,0.298362672328949
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,1023,0.009082666908701261
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,1023,0.008789333204428354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,2047,0.010960000256697336
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,4095,0.027050666511058807
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,4095,0.012986666212479273
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,8191,0.04464533428351084
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,16383,0.02722666660944621
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,8191,0.027050666511058807
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,16383,0.08063999811808269
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,32767,0.15210666259129843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,7,0.007061333085099856
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,32767,0.08226666847864787
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,1,0.0085333331177632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,65535,0.29715200265248615
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,65535,0.15568533539772034
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,255,0.0068693334857622785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,3,0.008037333066264788
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,3,0.007034666836261749
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,7,0.007674666742483775
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,7,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,15,0.00842666688064734
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,15,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,31,0.007893333211541176
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,31,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,63,0.008031999692320824
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,63,0.0069866664707660675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,127,0.008703999842206636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,127,0.006831999868154526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,16383,0.045738667249679565
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,255,0.008320000022649765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,255,0.006821333120266597
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,511,0.008762666955590248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,511,0.00860799973209699
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,1023,0.01090666651725769
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,2047,0.025424001117547352
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,4095,0.02646933247645696
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,4095,0.043354665239652
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,511,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,8191,0.08012266457080841
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,8191,0.04458666841189066
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,2047,0.010911999891201654
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,16383,0.0805920014778773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,32767,0.1530453364054362
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,32767,0.2955733338991801
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,1,0.007093333328763644
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,1,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,65535,0.2996799945831299
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,3,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,65535,0.5847733418146769
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,7,0.006746666505932808
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,7,0.006618666773041089
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,15,0.0069866664707660675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,15,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,31,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,31,0.00679466687142849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,63,0.0069386667261521024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,1023,0.00897066667675972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,63,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,127,0.006837333242098491
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,127,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,255,0.006773333375652631
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,2047,0.012757333616415659
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,511,0.008954666554927826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,511,0.007141333073377609
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,1023,0.008709333216150602
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,1023,0.008698666468262672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,1,0.007701333612203598
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,2047,0.011018666128317514
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,2047,0.008869333192706108
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,4095,0.012805332740147909
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,8191,0.02699733277161916
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,4095,0.010768000036478043
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,16383,0.1523413360118866
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,8191,0.013045333325862885
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,16383,0.04563733438650767
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,16383,0.02741333345572154
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,32767,0.0830080012480418
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,32767,0.04779199759165446
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,65535,0.15679466724395752
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,3,0.006858666737874349
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,3,0.006981333096822103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,65535,0.08437333504358928
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,7,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,7,0.006714666883150737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,3,0.006837333242098491
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,15,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,15,0.007231999809543292
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,31,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,31,0.00696000022192796
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,63,0.007002666592597961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,63,0.006720000257094701
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,127,0.00707733320693175
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,127,0.006805333619316419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,255,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,511,0.008682666967312494
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,511,0.00726400005320708
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,1023,0.008816000074148178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,1023,0.008762666955590248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,255,0.00666133314371109
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,2047,0.010901333143313726
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,2047,0.010858666151762009
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,4095,0.0269813338915507
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,4095,0.012741333494583765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,8191,0.04571733375390371
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,8191,0.0272533322374026
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,16383,0.046122665206591286
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,16383,0.08283199866612752
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,32767,0.15677866339683533
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,32767,0.08363200227419536
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,1,0.007280000175038974
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,1,0.007045333584149678
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,1,0.007189333438873291
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,1,0.009039999917149544
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,65535,0.301472008228302
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,3,0.008496000121037165
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,3,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,7,0.006725333631038666
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,7,0.008703999842206636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,15,0.006762666627764702
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,65535,0.15550399820009866
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,31,0.008752000207702318
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,31,0.006858666737874349
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,63,0.009872000043590864
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,63,0.006800000245372455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,255,0.0069386667261521024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,127,0.00877333308259646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,127,0.006650666395823161
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,255,0.008629333227872849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,255,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,511,0.00897066667675972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,2047,0.025621332228183746
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,2047,0.012826666235923767
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,1023,0.009002666920423508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,4095,0.043653334180514015
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,4095,0.02606933315594991
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,8191,0.08032000064849854
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,8191,0.044981335600217186
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,16383,0.0823413332303365
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,16383,0.15236799915631613
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,32767,0.29759466648101807
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,1,0.0069386667261521024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,3,0.007007999966541926
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,1,0.006698666761318843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,3,0.006693333387374878
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,15,0.00879466657837232
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,65535,0.5866613388061523
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,7,0.006671999891599019
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,15,0.0068853336075941724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,31,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,31,0.006746666505932808
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,15,0.006645333642760913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,63,0.00696000022192796
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,63,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,127,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,127,0.006682666639486949
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,255,0.0069866664707660675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,255,0.006688000013430913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,511,0.008954666554927826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,511,0.008613333106040955
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,511,0.007189333438873291
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,1023,0.008565333361426989
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,2047,0.008639999975760778
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,1023,0.012847999731699625
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,4095,0.00877333308259646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,4095,0.008752000207702318
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,8191,0.008821333448092142
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,8191,0.00915733352303505
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,16383,0.012863999853531519
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,16383,0.011109333485364914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,32767,0.016965333372354507
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,32767,0.015775999675194424
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,65535,0.02502399931351344
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,65535,0.02345066765944163
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,131071,0.02945599953333537
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,131071,0.027402666707833607
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,1,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,3,0.006805333619316419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,3,0.006746666505932808
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,7,0.007050666958093643
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,32767,0.15712533394495645
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,7,0.006693333387374878
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,15,0.007071999832987785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,31,0.007029333462317784
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,15,0.006773333375652631
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,31,0.006645333642760913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,63,0.006741333131988843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,63,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,127,0.00679466687142849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,127,0.006597333277265231
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,255,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,7,0.006677333265542984
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,255,0.006698666761318843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,65535,0.30187733968098956
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,511,0.008799999952316284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,1023,0.008816000074148178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,2047,0.008832000195980072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,2047,0.008821333448092142
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,4095,0.008565333361426989
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,4095,0.008863999818762144
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,8191,0.008901333436369896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,8191,0.008922666932145754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,16383,0.012719999998807907
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,1023,0.007061333085099856
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,2047,0.009935999910036722
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,16383,0.01414399966597557
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,32767,0.017551999539136887
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,32767,0.01695466662446658
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,65535,0.021189334491888683
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,65535,0.01937599976857503
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,131071,0.03939199944337209
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,131071,0.021151999632517498
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,1,0.00696000022192796
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,1,0.006805333619316419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,3,0.006762666627764702
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,3,0.006810666372378667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,7,0.006805333619316419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,7,0.006773333375652631
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,15,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,15,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,1,0.00666133314371109
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,31,0.006351999938488007
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,63,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,127,0.00701333334048589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,255,0.006624000146985054
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,255,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,511,0.00707733320693175
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,1023,0.008602666358153025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,1023,0.008586666857202848
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,511,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,1023,0.008672000219424566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,2047,0.008629333227872849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,2047,0.008565333361426989
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,4095,0.009002666920423508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,4095,0.008752000207702318
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,8191,0.008890666688481966
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,16383,0.013061333447694778
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,16383,0.012970666090647379
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,32767,0.01724799970785777
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,32767,0.015114666273196539
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,65535,0.03323733309904734
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,131071,0.051957334081331887
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,31,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,131071,0.03323733309904734
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,1,0.006805333619316419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,63,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,127,0.006762666627764702
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,3,0.007002666592597961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,3,0.006800000245372455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,7,0.006698666761318843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,7,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,15,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,15,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,511,0.00860799973209699
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,31,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,63,0.007087999954819679
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,63,0.006677333265542984
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,127,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,255,0.006730666384100914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,511,0.008549333239595095
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,511,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,1023,0.007647999872763951
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,2047,0.008634666601816813
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,2047,0.007344000041484833
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,4095,0.00874133345981439
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,4095,0.008757333581646284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,65535,0.01717866708834966
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,8191,0.008997333546479544
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,8191,0.00860799973209699
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,16383,0.012815999488035837
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,16383,0.012698666503032049
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,32767,0.0173333336909612
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,32767,0.015765332927306492
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,65535,0.021290667355060577
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,65535,0.019050666441520054
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,1,0.0069866664707660675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,131071,0.0383840004603068
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,131071,0.02144533395767212
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,1,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,1,0.006624000146985054
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,3,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,3,0.006698666761318843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,31,0.007007999966541926
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,7,0.006575999781489372
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,7,0.006704000135262807
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,15,0.007029333462317784
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,127,0.006810666372378667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,31,0.006677333265542984
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,255,0.006655999769767125
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,63,0.0069919998447100324
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,63,0.006720000257094701
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,127,0.007018666714429855
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,255,0.006981333096822103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,255,0.006501333167155583
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,511,0.008858666444818178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,511,0.006800000245372455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,1023,0.008767999708652496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,1023,0.008832000195980072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,8191,0.008816000074148178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,2047,0.00873066671192646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,4095,0.008943999807039896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,4095,0.008767999708652496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,8191,0.009525333220760027
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,8191,0.008767999708652496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,16383,0.014576000471909841
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,16383,0.013130666067202887
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,32767,0.01693333312869072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,32767,0.015541333705186844
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,65535,0.017162666966517765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,131071,0.052000001072883606
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,131071,0.033530667424201965
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,1,0.006981333096822103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,1,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,15,0.006768000001708667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,3,0.006981333096822103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,31,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,127,0.0069226666043202085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,3,0.007002666592597961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,7,0.007018666714429855
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,7,0.006677333265542984
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,1023,0.008885333314538002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,15,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,31,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,15,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,31,0.006757333253820737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,63,0.006768000001708667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,63,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,127,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,2047,0.008698666468262672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,255,0.006751999879876773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,255,0.006757333253820737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,511,0.00731733317176501
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,511,0.009002666920423508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,2047,0.008933333059151968
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,1023,0.008757333581646284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,2047,0.008661333471536636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,4095,0.00892800030608972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,8191,0.012837332983811697
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,8191,0.010981333752473196
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,16383,0.014912000546852747
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,16383,0.013210666676362356
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,32767,0.015392000476519266
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,65535,0.048623998959859215
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,65535,0.0335413341720899
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,131071,0.08462400237719218
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,131071,0.05011733373006185
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,1,0.006709333509206772
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,1,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,3,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,3,0.006714666883150737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,7,0.006831999868154526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,7,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,15,0.0069919998447100324
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,15,0.006698666761318843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,31,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,31,0.006725333631038666
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,63,0.007007999966541926
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,127,0.006629333520929019
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,127,0.006762666627764702
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,127,0.006751999879876773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,1023,0.00891733355820179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,255,0.007157333195209503
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,255,0.006768000001708667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,4095,0.008778666456540426
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,511,0.008693333094318708
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,511,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,1023,0.009557333464423815
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,32767,0.03014400104681651
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,2047,0.011034666250149408
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,65535,0.03108799954255422
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,2047,0.010847999403874079
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,8191,0.045567999283472695
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,4095,0.027077332139015198
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,4095,0.012885333349307379
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,16383,0.08230933547019958
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,8191,0.027301333844661713
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,16383,0.046581332882245384
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,1,0.008912000184257826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,32767,0.15787733594576517
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,3,0.008656000097592672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,1,0.00744000015159448
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,3,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,7,0.008613333106040955
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,32767,0.08262933293978374
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,7,0.006751999879876773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,15,0.00891733355820179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,31,0.006784000123540561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,15,0.007829333345095316
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,63,0.00860799973209699
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,63,0.007002666592597961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,63,0.007002666592597961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,31,0.008943999807039896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,127,0.007184000064929326
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,255,0.006682666639486949
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,511,0.009002666920423508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,511,0.008826666822036108
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,255,0.009242666885256767
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,1023,0.008896000062425932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,1023,0.01146666705608368
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,1023,0.008698666468262672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,2047,0.025968000292778015
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,2047,0.014005333185195923
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,4095,0.044906665881474815
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,4095,0.02740799884001414
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,8191,0.045221333702405296
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,8191,0.08297066887219746
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,1,0.008736000085870424
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,16383,0.15541332960128784
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,3,0.008954666554927826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,1,0.009381333366036415
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,3,0.008661333471536636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,7,0.009141333401203156
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,32767,0.30274667342503864
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,127,0.008933333059151968
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,32767,0.15598400433858237
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,7,0.009296000003814697
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,15,0.00873066671192646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,15,0.008816000074148178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,31,0.00892800030608972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,63,0.009312000125646591
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,127,0.009029333169261614
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,127,0.008746666833758354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,255,0.008746666833758354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,31,0.008949333180983862
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,63,0.009018666421373686
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,511,0.010608000059922537
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,2047,0.02731200059254964
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,1023,0.012815999488035837
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,511,0.013023999830087027
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,4095,0.04565866788228353
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,4095,0.08060800035794576
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,16383,0.0824480007092158
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,8191,0.08334933718045552
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,8191,0.15404799580574036
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,16383,0.29893867174784344
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,1,0.007061333085099856
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,16383,0.15818132956822714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,3,0.006757333253820737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,1,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,3,0.006618666773041089
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,7,0.006821333120266597
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,15,0.006640000268816948
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,15,0.006800000245372455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,255,0.008986666798591614
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,31,0.0069226666043202085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,31,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,32767,0.591920018196106
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,63,0.006735999758044879
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,127,0.007173333317041397
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,32767,0.31092266241709393
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,255,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,255,0.006789333497484525
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,511,0.00707733320693175
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,1023,0.008976000050703684
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,2047,0.044309332966804504
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,127,0.00697066696981589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,1023,0.008832000195980072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,4095,0.008954666554927826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,511,0.009306666751702627
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,4095,0.008922666932145754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,2047,0.00850133349498113
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,8191,0.009535999968647957
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,16383,0.013306666165590286
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,8191,0.009488000224033991
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,16383,0.013232000172138214
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,32767,0.015279999623696009
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,65535,0.03418133407831192
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,65535,0.017418666432301205
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,131071,0.051407997806866966
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,1,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,1,0.006789333497484525
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,3,0.006805333619316419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,3,0.006837333242098491
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,7,0.007104000076651573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,7,0.006858666737874349
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,7,0.00730666642387708
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,15,0.006528000036875407
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,31,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,31,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,63,0.006704000135262807
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,63,0.006560000280539195
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,63,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,127,0.006735999758044879
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,127,0.006805333619316419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,255,0.006746666505932808
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,511,0.008757333581646284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,511,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,1023,0.008901333436369896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,1023,0.027066667874654133
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,1023,0.008725333337982496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,2047,0.008858666444818178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,2047,0.008789333204428354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,4095,0.00871999996403853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,4095,0.0086666668454806
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,8191,0.01312000056107839
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,2047,0.008767999708652496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,8191,0.01109333336353302
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,16383,0.015146666516860327
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,16383,0.013253333667914072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,32767,0.017258666455745697
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,32767,0.03121600051720937
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,32767,0.01594666639963786
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,65535,0.049322664737701416
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,65535,0.03138133386770884
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,131071,0.049733335773150124
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,131071,0.0867039958635966
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,1,0.006768000001708667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,131071,0.03410666684309641
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,3,0.007269333427151044
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,7,0.006746666505932808
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,7,0.007135999699433644
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,15,0.006560000280539195
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,15,0.00725333330531915
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,15,0.00702400008837382
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,31,0.006757333253820737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,31,0.007418666655818622
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,63,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,63,0.007226666435599327
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,127,0.007135999699433644
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,127,0.007141333073377609
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,255,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,255,0.007189333438873291
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,511,0.008842666943868002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,1023,0.009712000067035357
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,511,0.007184000064929326
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,255,0.006730666384100914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,2047,0.008912000184257826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,1023,0.009269333134094873
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,2047,0.008778666456540426
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,4095,0.010709332923094431
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,4095,0.011247999966144562
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,8191,0.014954666296641031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,16383,0.02922666569550832
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,8191,0.013221333424250284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,16383,0.01481066644191742
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,32767,0.029306667546431225
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,32767,0.04842666784922282
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,65535,0.08258666594823201
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,65535,0.04837333162625631
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,131071,0.15537066260973612
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,131071,0.08571733037630717
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,1,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,1,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,1,0.009002666920423508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,3,0.006789333497484525
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,3,0.00666133314371109
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,3,0.008896000062425932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,7,0.007093333328763644
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,15,0.008586666857202848
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,15,0.007125333572427432
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,31,0.006735999758044879
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,63,0.008672000219424566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,31,0.008842666943868002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,127,0.00891733355820179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,127,0.007151999821265538
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,63,0.007269333427151044
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,255,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,511,0.00901333304742972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,511,0.008901333436369896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,1023,0.00891733355820179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,1023,0.011237333218256632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,2047,0.012709333250919977
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,2047,0.02731200059254964
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,4095,0.04568533102671305
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,8191,0.08287466565767924
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,4095,0.02736533433198929
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,7,0.008853333070874214
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,8191,0.04614933331807455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,1,0.00891733355820179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,16383,0.15780267119407654
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,3,0.008767999708652496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,7,0.009008000294367472
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,7,0.009072000160813332
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,1,0.009098666409651438
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,255,0.008623999853928884
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,16383,0.0831413318713506
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,15,0.008821333448092142
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,3,0.008965333302815756
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,15,0.008837333569924036
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,31,0.009189333145817121
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,31,0.008853333070874214
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,63,0.00901333304742972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,63,0.008778666456540426
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,127,0.00873066671192646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,255,0.008842666943868002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,255,0.00919999989370505
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,127,0.008901333436369896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,511,0.012576000144084295
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,511,0.01081066702802976
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,1023,0.012858666479587555
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,4095,0.045696000258127846
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,4095,0.08263466755549113
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,1023,0.02644266684850057
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,2047,0.027050666511058807
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,2047,0.04479999840259552
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,1,0.011968000481526056
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,8191,0.08387200037638347
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,8191,0.15668267011642456
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,1,0.011034666250149408
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,3,0.011029332876205444
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,16383,0.1569973329703013
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,7,0.013050666699806849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,15,0.011813333878914515
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,7,0.01097600037852923
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,3,0.011711999773979187
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,16383,0.3028906583786011
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,15,0.010949333508809408
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,31,0.010837333897749582
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,63,0.012181332955757776
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,63,0.010938666760921478
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,31,0.012597333639860153
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,127,0.01157333329319954
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,255,0.013007999708255133
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,255,0.010970667004585266
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,127,0.010890666395425797
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,511,0.013658666362365087
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,511,0.026752000053723652
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,2047,0.0819893330335617
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,2047,0.045834665497144066
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,1023,0.027471999327341717
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,1023,0.0452159990866979
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,4095,0.15594133734703064
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,8191,0.15732266505559286
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,1,0.008767999708652496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,4095,0.08358933528264363
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,8191,0.3065226674079895
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,1,0.009338666374484697
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,3,0.00903466654320558
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,3,0.008725333337982496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,7,0.008912000184257826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,7,0.00891733355820179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,16383,0.5942080020904541
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,16383,0.30779733260472614
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,15,0.009072000160813332
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,31,0.008874666566650072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,31,0.009002666920423508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,63,0.008826666822036108
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,63,0.008714666590094566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,255,0.008682666967312494
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,255,0.008650666723648706
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,511,0.012549333274364471
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,511,0.010842667271693548
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,1023,0.027087998886903126
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,1023,0.012784000486135483
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,127,0.009008000294367472
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,127,0.008746666833758354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,15,0.009205333267649015
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,2047,0.02758399893840154
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,2047,0.045594667394955955
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,1,0.01293333371480306
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,1,0.010757333288590113
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,3,0.01231466606259346
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,3,0.01099733387430509
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,4095,0.04776533444722494
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,4095,0.08261866867542267
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,15,0.011909333368142446
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,15,0.010837333897749582
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,7,0.011989332735538483
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,7,0.010698666175206503
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,31,0.01309866706530253
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,127,0.012565333396196365
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,127,0.01089599976936976
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,255,0.012074666718641916
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,255,0.010890666395425797
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,511,0.027274665733178455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,511,0.01314666618903478
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,63,0.012432000289360682
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,63,0.010768000036478043
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,1023,0.04574400186538696
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,31,0.011055999745925268
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,1023,0.027674667537212372
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,4095,0.15631999572118124
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,4095,0.08449066678682964
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,1,0.017029333859682083
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,2047,0.08297066887219746
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,3,0.017162666966517765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,1,0.014970666418472925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,7,0.016943999876578648
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,3,0.014991999914248785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,7,0.014783999572197596
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,15,0.017024000485738117
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,31,0.016821333517630894
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,31,0.01492799942692121
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,63,0.01699200024207433
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,63,0.015029333531856537
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,15,0.015168000012636185
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,127,0.015024000157912573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,127,0.016842667013406754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,255,0.026181332767009735
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,2047,0.04614399870236715
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,255,0.01481066644191742
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,2047,0.15153599778811136
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,511,0.0444106658299764
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,511,0.027130665878454845
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,2047,0.08252800007661183
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,1023,0.04539200166861216
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,1023,0.08052800099054973
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,1,0.012485332787036896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,4095,0.15915200114250183
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,1,0.010879999647537867
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,3,0.012970666090647379
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,4095,0.29733333985010785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,7,0.010837333897749582
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,15,0.012773333738247553
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,15,0.010826667149861654
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,3,0.010933333386977514
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,31,0.012432000289360682
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,7,0.012949333836634954
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,31,0.011045332998037338
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,63,0.012879999975363413
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,63,0.0107893335322539
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,127,0.012949333836634954
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,127,0.010863999525705973
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,255,0.012346666306257248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,255,0.011018666128317514
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,511,0.027290667096773785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,511,0.014186666657527288
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,1023,0.045498669147491455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,1023,0.027509334186712902
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,1,0.017125333348910015
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,1,0.015018666783968607
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,3,0.016949333250522614
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,2047,0.08412266770998637
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,2047,0.04711999992529551
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,3,0.015247999380032221
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,7,0.016917333006858826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,15,0.017077332983414333
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,15,0.015034666905800501
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,7,0.014922666052977243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,31,0.01720533271630605
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,63,0.014874666929244995
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,127,0.01684800038735072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,127,0.015098666151364645
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,255,0.026496000587940216
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,255,0.014730667074521383
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,511,0.04498666524887085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,511,0.027493332823117573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,63,0.01695999999841054
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,31,0.01498666654030482
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,1023,0.08202133576075236
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,3,0.023189333577950794
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,3,0.027642667293548584
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,1023,0.04545066754023234
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,1,0.02749866743882497
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,1,0.023285334308942158
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,7,0.02311466634273529
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,15,0.0271573339899381
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,2047,0.08433600266774495
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,2047,0.15518933534622192
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,31,0.027450665831565857
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,31,0.023205332458019257
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,15,0.023200000325838726
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,63,0.02720533311367035
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,63,0.022944000860055287
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,7,0.027109332382678986
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,255,0.028399998943010967
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,511,0.08052266637484233
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,255,0.0454773356517156
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,511,0.045791998505592346
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,1,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,2047,0.29636265834172565
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,3,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,7,0.007082666580875714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,127,0.023391999304294586
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,127,0.029301332930723827
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,7,0.006634666894872983
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,3,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,31,0.00666133314371109
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,1023,0.08242666721343994
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,63,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,1023,0.15229333440462747
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,127,0.006677333265542984
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,255,0.006709333509206772
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,15,0.006784000123540561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,1,0.007002666592597961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,15,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,255,0.006751999879876773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,511,0.008613333106040955
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,511,0.006826666494210561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,1023,0.008976000050703684
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,1023,0.008757333581646284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,2047,0.1569919983545939
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,2047,0.00879466657837232
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,2047,0.008693333094318708
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,4095,0.008698666468262672
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,4095,0.008997333546479544
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,8191,0.010666667173306147
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,16383,0.014943999548753103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,16383,0.01312000056107839
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,32767,0.029717333614826202
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,65535,0.048826664686203
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,8191,0.013034666577974955
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,65535,0.031248000760873158
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,1,0.006698666761318843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,131071,0.051327998439470925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,1,0.006549333532651265
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,3,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,3,0.0064266665528217954
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,7,0.00696000022192796
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,7,0.006698666761318843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,15,0.007093333328763644
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,15,0.006757333253820737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,31,0.006800000245372455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,31,0.006650666395823161
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,63,0.0069386667261521024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,63,0.006688000013430913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,127,0.006805333619316419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,127,0.006613333399097125
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,255,0.006677333265542984
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,63,0.006597333277265231
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,255,0.006650666395823161
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,511,0.0086666668454806
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,511,0.007514666765928268
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,1023,0.008879999940594038
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,1023,0.008778666456540426
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,2047,0.008799999952316284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,2047,0.00871999996403853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,4095,0.01108266661564509
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,4095,0.010656000425418219
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,8191,0.013909333695967993
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,8191,0.012693333129088083
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,32767,0.015247999380032221
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,16383,0.029093332588672638
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,16383,0.014901333798964819
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,32767,0.04791999856630961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,32767,0.029498666524887085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,65535,0.04865066707134247
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,131071,0.08621866504351298
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,1,0.006837333242098491
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,131071,0.15754666924476624
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,131071,0.08669867118199666
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,3,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,3,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,7,0.006981333096822103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,7,0.006682666639486949
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,15,0.00706666645904382
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,31,0.006634666894872983
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,15,0.006746666505932808
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,31,0.006666666517655055
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,31,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,63,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,63,0.006821333120266597
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,127,0.007007999966541926
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,127,0.006725333631038666
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,255,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,255,0.00706666645904382
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,511,0.00871999996403853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,511,0.0069973332186539965
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,1023,0.00897066667675972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,1023,0.00879466657837232
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,127,0.0068693334857622785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,2047,0.010751999914646149
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,2047,0.008922666932145754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,4095,0.013183999806642532
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,8191,0.02740799884001414
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,16383,0.027295999228954315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,16383,0.045509333411852516
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,8191,0.013354666531085968
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,32767,0.04569066564242045
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,32767,0.08101333181063335
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,65535,0.08509866396586101
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,65535,0.15398400028546652
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,65535,0.0851146678129832
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,131071,0.2980159918467204
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,131071,0.15793066223462424
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,1,0.007120000198483467
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,4095,0.010751999914646149
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,1,0.015034666905800501
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,3,0.01693333312869072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,3,0.014682666709025701
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,7,0.015109332899252573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,15,0.017103999853134155
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,1,0.01747200017174085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,15,0.014741333822409311
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,7,0.017423999806245167
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,31,0.01691199963291486
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,31,0.015146666516860327
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,63,0.016800000021855038
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,63,0.014975999792416891
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,127,0.01721599946419398
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,255,0.027141332626342773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,511,0.04544533292452494
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,127,0.014752000570297241
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,255,0.014725333700577417
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,511,0.027295999228954315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,1,0.027376001079877216
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,1023,0.0823413332303365
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,1023,0.04640533526738485
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,1,0.023039999107519787
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,3,0.023077333966890972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,3,0.027386667827765148
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,7,0.027274665733178455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,15,0.02738133321205775
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,7,0.023285334308942158
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,31,0.022885332504908245
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,15,0.023045333723227184
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,31,0.02739199995994568
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,63,0.027114666998386383
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,63,0.022895999252796173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,127,0.02908266584078471
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,127,0.023226665953795116
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,255,0.045696000258127846
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,255,0.028762665887673695
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,511,0.08197866876920064
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,511,0.04606399933497111
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,1,0.04779199759165446
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,1,0.039605334401130676
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,1023,0.15635200341542563
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,1023,0.08252800007661183
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,3,0.04752533137798309
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,3,0.03965333352486292
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,7,0.047839999198913574
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,15,0.04780800143877665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,7,0.039594667653242745
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,15,0.03945599993069967
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,63,0.04914666712284088
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,63,0.039813332259655
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,127,0.05041066805521647
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,127,0.03977066775163015
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,255,0.0804319977760315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,255,0.04823466638724009
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,31,0.049653331438700356
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,31,0.03948266555865606
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,511,0.0823520024617513
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,511,0.15224533279736838
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,1023,0.15597866972287497
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,1,0.02731200059254964
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,3,0.023269332945346832
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,3,0.027488000690937042
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,7,0.02734400083621343
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,1023,0.295632004737854
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,7,0.023077333966890972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,1,0.02313599983851115
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,15,0.02330133318901062
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,31,0.027429332335789997
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,31,0.023018665611743927
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,15,0.027434666951497395
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,63,0.027072000006834667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,127,0.03046933313210805
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,127,0.023103999594847362
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,255,0.045824001232783
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,63,0.02332799881696701
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,511,0.08278400202592213
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,255,0.02903999884923299
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,511,0.047055999437967934
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,1,0.048112000028292336
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,1,0.039493332306543984
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,3,0.04868266483147939
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,3,0.03975466638803482
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,7,0.04857600231965383
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,7,0.03944533318281174
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,15,0.048528000712394714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,31,0.048021331429481506
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,15,0.039664000272750854
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,31,0.039749334255854286
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,63,0.04782933493455251
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,63,0.0394400010506312
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,127,0.05195199946562449
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,127,0.04075733323891958
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,255,0.0491946687301
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,255,0.08241599798202515
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,1,0.0727893312772115
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,1,0.08804800113042195
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,511,0.08388266960779826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,511,0.15692266821861267
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,3,0.08941866954167683
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,3,0.072202667593956
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,7,0.07247999807198842
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,7,0.08847999572753906
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,15,0.08851200342178345
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,15,0.07231466472148895
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,63,0.0886400043964386
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,63,0.07232533395290375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,31,0.0888320008913676
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,31,0.07281599938869476
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,127,0.07318399846553802
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,1,0.007034666836261749
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,3,0.006773333375652631
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,1,0.006805333619316419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,7,0.006810666372378667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,7,0.0069386667261521024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,15,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,15,0.006773333375652631
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,31,0.006826666494210561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,31,0.007087999954819679
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,63,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,63,0.006831999868154526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,127,0.006858666737874349
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,127,0.006768000001708667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,255,0.007034666836261749
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,255,0.007007999966541926
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,3,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,511,0.008687999720374743
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,511,0.007151999821265538
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,1023,0.008885333314538002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,1023,0.0086666668454806
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,127,0.09225066502888997
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,2047,0.008842666943868002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,2047,0.008725333337982496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,4095,0.011045332998037338
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,4095,0.010741333166758219
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,8191,0.01302933320403099
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,16383,0.029264000554879505
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,16383,0.014954666296641031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,32767,0.04771199822425842
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,32767,0.029370665550231934
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,65535,0.08604266246159871
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,255,0.1532266636689504
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,255,0.08866133292516072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,65535,0.04923733572165171
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,8191,0.014746667196353277
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,131071,0.1584160029888153
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,3,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,131071,0.08603733777999878
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,3,0.00666133314371109
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,7,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,7,0.006800000245372455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,15,0.006725333631038666
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,31,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,31,0.006666666517655055
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,63,0.006821333120266597
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,63,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,127,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,127,0.0069919998447100324
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,255,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,255,0.006741333131988843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,511,0.008570666735370954
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,511,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,1023,0.008837333569924036
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,1023,0.008650666723648706
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,2047,0.01071999967098236
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,2047,0.0086666668454806
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,4095,0.012847999731699625
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,4095,0.010960000256697336
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,8191,0.027424000203609467
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,8191,0.013002666334311167
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,1,0.007082666580875714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,16383,0.045824001232783
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,16383,0.02752000093460083
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,1,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,32767,0.08265600105126698
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,32767,0.04619733492533366
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,15,0.0069866664707660675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,65535,0.08366400003433228
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,65535,0.1564853290716807
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,1,0.007231999809543292
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,1,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,131071,0.30193066596984863
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,3,0.006688000013430913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,7,0.007130666946371396
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,3,0.006730666384100914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,131071,0.15800533692042032
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,7,0.00701333334048589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,15,0.006826666494210561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,15,0.0069226666043202085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,31,0.007109333450595538
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,31,0.007567999884486198
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,63,0.00684799998998642
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,511,0.2966826756795247
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,63,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,255,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,255,0.006831999868154526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,511,0.008714666590094566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,511,0.00874133345981439
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,1023,0.00921066664159298
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,1023,0.008746666833758354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,2047,0.010816000401973724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,4095,0.027221334477265675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,8191,0.027306665976842243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,4095,0.012847999731699625
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,8191,0.04471466441949209
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,16383,0.04580800235271454
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,16383,0.08099199831485748
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,32767,0.15411733587582907
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,32767,0.08330133557319641
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,65535,0.29809067646662396
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,65535,0.15582399566968283
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,1,0.006698666761318843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,131071,0.5848319927851359
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,1,0.0064853330453236895
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,3,0.0068853336075941724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,127,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,7,0.0068693334857622785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,7,0.006826666494210561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,3,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,15,0.006858666737874349
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,15,0.006618666773041089
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,511,0.15615466237068176
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,31,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,31,0.006650666395823161
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,63,0.006650666395823161
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,63,0.007007999966541926
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,127,0.006784000123540561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,127,0.0068853336075941724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,255,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,255,0.006666666517655055
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,511,0.007146666447321574
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,1023,0.008757333581646284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,2047,0.011306667079528173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,2047,0.008832000195980072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,4095,0.013082666943470636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,4095,0.010837333897749582
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,8191,0.027210667729377747
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,8191,0.012885333349307379
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,131071,0.3030613263448079
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,16383,0.04568000137805939
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,16383,0.027285332481066387
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,127,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,32767,0.04600533346335093
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,32767,0.08339200417200725
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,65535,0.08586133519808452
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,1,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,65535,0.15691733360290527
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,3,0.007349333415428798
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,1023,0.008538666491707167
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,2047,0.010485333700974783
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,7,0.007402666533986728
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,3,0.007397333160042763
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,131071,0.15965867042541504
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,7,0.00702400008837382
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,31,0.006757333253820737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,131071,0.3044319947560628
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,63,0.00702400008837382
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,63,0.006805333619316419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,31,0.006762666627764702
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,511,0.008778666456540426
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,127,0.0069866664707660675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,127,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,255,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,255,0.006757333253820737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,511,0.0074346667776505155
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,1023,0.009829333052039146
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,1023,0.00890666681031386
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,2047,0.011242666592200598
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,2047,0.010650667051474253
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,4095,0.026975999275843304
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,4095,0.012981332838535309
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,8191,0.04574933151404063
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,8191,0.02718399961789449
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,16383,0.04543466866016388
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,1,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,32767,0.0827466646830241
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,32767,0.15680533647537231
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,65535,0.3023573358853658
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,15,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,65535,0.1566986640294393
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,131071,0.3044000069300334
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,1,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,7,0.007360000163316727
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,511,0.00860799973209699
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,3,0.006762666627764702
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,7,0.007029333462317784
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,15,0.006757333253820737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,15,0.006650666395823161
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,31,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,131071,0.5955520073572794
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,31,0.006634666894872983
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,63,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,127,0.006575999781489372
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,255,0.00696000022192796
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,127,0.006751999879876773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,255,0.00660800002515316
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,511,0.008576000109314919
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,511,0.006720000257094701
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,1023,0.00871999996403853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,1023,0.008837333569924036
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,2047,0.00860799973209699
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,2047,0.007242666557431221
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,4095,0.008853333070874214
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,4095,0.00854399986565113
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,8191,0.00897066667675972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,8191,0.008858666444818178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,16383,0.013056000073750814
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,16383,0.010826667149861654
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,32767,0.01693333312869072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,32767,0.01515199989080429
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,65535,0.025072000920772552
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,65535,0.023215999205907185
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,131071,0.029605334003766377
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,131071,0.02720533311367035
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,1,0.007055999711155891
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,1,0.006650666395823161
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,3,0.00696000022192796
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,15,0.006778666749596596
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,1,0.006725333631038666
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,7,0.007029333462317784
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,7,0.0064213331788778305
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,15,0.007034666836261749
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,15,0.006704000135262807
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,31,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,31,0.006709333509206772
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,63,0.006773333375652631
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,63,0.006735999758044879
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,127,0.006565333033601443
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,127,0.006773333375652631
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,255,0.0069226666043202085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,255,0.00679466687142849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,511,0.00873066671192646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,511,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,1023,0.008661333471536636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,2047,0.008863999818762144
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,2047,0.008725333337982496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,4095,0.008896000062425932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,63,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,8191,0.008997333546479544
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,8191,0.00895999992887179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,16383,0.012938667088747025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,16383,0.08285333216190338
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,16383,0.012997332960367203
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,32767,0.01718933383623759
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,65535,0.021402666966120403
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,65535,0.021146667500336964
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,131071,0.03931200007597605
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,131071,0.021189334491888683
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,1,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,1,0.00679466687142849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,3,0.007167999943097432
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,3,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,7,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,7,0.006837333242098491
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,15,0.006800000245372455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,15,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,31,0.006800000245372455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,31,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,63,0.0069226666043202085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,63,0.006821333120266597
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,127,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,127,0.006464000170429547
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,255,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,255,0.0069226666043202085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,511,0.008623999853928884
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,511,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,1023,0.008949333180983862
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,1023,0.008682666967312494
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,1023,0.00855466661353906
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,2047,0.00901333304742972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,2047,0.008709333216150602
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,4095,0.008896000062425932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,4095,0.008938666433095932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,4095,0.008687999720374743
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,8191,0.009018666421373686
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,8191,0.008943999807039896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,16383,0.012975999464591345
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,16383,0.012752000242471695
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,32767,0.016976000120242436
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,32767,0.016672000288963318
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,65535,0.021231998999913532
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,65535,0.01932799940307935
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,131071,0.039018665750821434
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,131071,0.02107200026512146
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,1,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,1,0.0063786668082078295
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,3,0.006655999769767125
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,3,0.006784000123540561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,7,0.006778666749596596
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,7,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,15,0.006773333375652631
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,15,0.006831999868154526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,31,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,31,0.006858666737874349
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,63,0.006741333131988843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,127,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,255,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,255,0.006709333509206772
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,511,0.008922666932145754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,511,0.006826666494210561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,1023,0.008842666943868002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,1023,0.008869333192706108
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,2047,0.008922666932145754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,2047,0.00878399983048439
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,4095,0.008922666932145754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,4095,0.008922666932145754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,3,0.006709333509206772
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,8191,0.009039999917149544
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,8191,0.009002666920423508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,16383,0.013658666362365087
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,16383,0.013013333082199097
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,32767,0.017136000096797943
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,32767,0.01505600040157636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,65535,0.03316266586383184
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,65535,0.01717866708834966
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,131071,0.05226133267084757
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,1,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,1,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,3,0.007018666714429855
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,3,0.007087999954819679
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,3,0.006762666627764702
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,7,0.0069386667261521024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,7,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,15,0.006789333497484525
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,15,0.006810666372378667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,31,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,63,0.00697066696981589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,32767,0.016837333639462788
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,63,0.006746666505932808
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,127,0.007082666580875714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,127,0.006645333642760913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,63,0.006704000135262807
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,255,0.007029333462317784
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,255,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,511,0.008639999975760778
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,511,0.00772266648709774
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,1023,0.009237333511312803
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,1023,0.00891733355820179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,2047,0.012896000097195307
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,2047,0.010634666929642359
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,4095,0.02696000039577484
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,4095,0.012960000584522883
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,8191,0.04542933404445648
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,131071,0.03367999941110611
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,8191,0.027093333502610523
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,16383,0.08398933211962382
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,32767,0.08256533245245616
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,16383,0.045978665351867676
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,1,0.007471999774376552
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,1,0.006805333619316419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,127,0.007104000076651573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,65535,0.30457067489624023
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,3,0.009056000038981438
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,65535,0.15788267056147257
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,3,0.006981333096822103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,7,0.008842666943868002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,15,0.008570666735370954
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,31,0.00860799973209699
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,31,0.007034666836261749
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,7,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,15,0.007029333462317784
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,63,0.008885333314538002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,63,0.00660800002515316
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,127,0.008842666943868002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,255,0.008527999743819237
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,255,0.0069919998447100324
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,511,0.008933333059151968
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,511,0.008709333216150602
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,1023,0.010863999525705973
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,1023,0.008976000050703684
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,2047,0.02736533433198929
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,2047,0.01301866645614306
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,4095,0.04410133262475332
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,31,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,4095,0.02735466758410136
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,8191,0.08251733581225078
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,16383,0.08256533245245616
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,8191,0.045610666275024414
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,1,0.006730666384100914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,16383,0.15506666898727417
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,32767,0.3027999997138977
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,1,0.006645333642760913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,32767,0.15557866295178732
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,3,0.007125333572427432
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,3,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,127,0.006650666395823161
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,15,0.007733333234985669
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,31,0.007055999711155891
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,65535,0.3094346721967061
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,31,0.006693333387374878
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,63,0.007018666714429855
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,7,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,127,0.0069386667261521024
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,65535,0.5956533352533976
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,127,0.006741333131988843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,255,0.00679466687142849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,32767,0.1576640009880066
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,7,0.00666133314371109
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,511,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,1023,0.008874666566650072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,2047,0.008650666723648706
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,2047,0.008629333227872849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,1023,0.008672000219424566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,4095,0.008943999807039896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,4095,0.008581333483258883
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,8191,0.010144000252087912
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,8191,0.00874133345981439
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,16383,0.013327999661366144
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,511,0.008703999842206636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,16383,0.012757333616415659
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,32767,0.01717866708834966
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,32767,0.015471999843915304
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,65535,0.034389334420363106
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,65535,0.01721599946419398
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,131071,0.05173333485921224
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,131071,0.03368533402681351
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,1,0.006800000245372455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,3,0.006906666482488315
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,3,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,7,0.006805333619316419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,7,0.006624000146985054
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,15,0.00697066696981589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,15,0.0068853336075941724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,31,0.006810666372378667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,31,0.006677333265542984
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,63,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,63,0.0068853336075941724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,127,0.006858666737874349
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,127,0.00666133314371109
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,255,0.006981333096822103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,255,0.007344000041484833
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,511,0.008832000195980072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,511,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,1023,0.00867733359336853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,1023,0.008816000074148178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,2047,0.0085333331177632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,2047,0.008709333216150602
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,4095,0.008858666444818178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,4095,0.00854399986565113
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,8191,0.012800000607967377
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,8191,0.010879999647537867
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,15,0.0064106664309899015
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,16383,0.013023999830087027
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,32767,0.03102933367093404
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,32767,0.015173333386580149
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,65535,0.048826664686203
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,65535,0.03218133250872294
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,255,0.006784000123540561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,131071,0.08614400029182434
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,63,0.006725333631038666
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,1,0.008474666625261307
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,1,0.006751999879876773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,3,0.008682666967312494
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,3,0.006826666494210561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,1,0.006831999868154526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,7,0.00879466657837232
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,7,0.007157333195209503
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,15,0.008661333471536636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,15,0.006810666372378667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,31,0.00890666681031386
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,31,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,63,0.008943999807039896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,63,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,127,0.008682666967312494
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,127,0.006858666737874349
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,255,0.00855466661353906
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,255,0.006949333474040031
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,16383,0.015013333410024643
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,511,0.008933333059151968
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,511,0.008714666590094566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,1023,0.011034666250149408
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,131071,0.0498986691236496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,1023,0.008943999807039896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,2047,0.013072000195582708
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,8191,0.08306666711966197
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,2047,0.02605866640806198
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,8191,0.04578666885693868
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,16383,0.15633599956830344
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,4095,0.02698666602373123
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,16383,0.08264000217119853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,4095,0.045040001471837364
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,1,0.008992000172535578
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,1,0.008938666433095932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,3,0.009290666629870733
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,7,0.008986666798591614
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,15,0.009253333633144697
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,15,0.009141333401203156
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,31,0.009359999870260557
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,7,0.009226666763424873
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,3,0.00895999992887179
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,31,0.0085333331177632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,32767,0.15652799606323242
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,63,0.009008000294367472
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,63,0.009039999917149544
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,127,0.009402666861812273
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,127,0.008757333581646284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,255,0.009039999917149544
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,255,0.008767999708652496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,511,0.010591999938090643
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,32767,0.30318933725357056
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,1023,0.01293333371480306
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,1023,0.02643200010061264
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,511,0.01293333371480306
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,4095,0.0823466678460439
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,4095,0.04569066564242045
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,2047,0.027482666075229645
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,8191,0.15652799606323242
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,2047,0.04545066754023234
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,8191,0.08404800295829773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,1,0.008799999952316284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,32767,0.3113759954770406
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,16383,0.16061333815256754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,16383,0.3039360046386719
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,1,0.009045333291093508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,3,0.009008000294367472
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,7,0.008943999807039896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,32767,0.6006453434626261
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,15,0.009103999783595404
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,31,0.009216000015536943
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,31,0.008634666601816813
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,63,0.008954666554927826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,15,0.008901333436369896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,7,0.009162666896979014
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,127,0.009125333279371262
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,127,0.008778666456540426
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,255,0.008943999807039896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,255,0.008976000050703684
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,511,0.011685332904259363
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,3,0.00877333308259646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,511,0.01128000020980835
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,1023,0.026629333694775898
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,63,0.00915733352303505
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,2047,0.04566933214664459
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,2047,0.027280000348885853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,8191,0.15676800409952799
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,4095,0.0823520024617513
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,8191,0.08369599779446919
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,1,0.012517333030700684
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,1,0.010992000500361124
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,4095,0.048351998130480446
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,3,0.012341332932313284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,1023,0.012885333349307379
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,7,0.010735999792814255
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,15,0.012383999923865
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,3,0.010965333630641302
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,7,0.013248000293970108
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,31,0.011711999773979187
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,31,0.011007999380429586
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,63,0.013072000195582708
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,63,0.010869332899649939
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,127,0.012549333274364471
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,255,0.012202666451533636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,127,0.01073066641887029
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,255,0.010656000425418219
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,511,0.02701333413521449
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,511,0.013162666310866674
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,15,0.010863999525705973
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,1023,0.04584000011285146
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,4095,0.15659733613332114
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,4095,0.0839466651280721
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,8191,0.1562986671924591
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,2047,0.08273600041866302
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,2047,0.04560533165931702
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,8191,0.30692267417907715
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,1,0.012928000340859095
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,1,0.010837333897749582
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,3,0.01257066677014033
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,3,0.011136000355084738
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,1023,0.02719466636578242
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,15,0.012154666086037954
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,15,0.010874666273593903
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,31,0.012842666357755661
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,31,0.010629333555698395
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,7,0.012901333471139273
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,63,0.012960000584522883
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,7,0.010826667149861654
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,63,0.010890666395425797
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,127,0.012437333663304647
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,127,0.010922666639089584
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,255,0.01173866664369901
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,255,0.010709332923094431
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,511,0.026885333160559338
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,511,0.014479999740918478
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,1023,0.04550399879614512
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,1023,0.027215999861558277
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,1,0.016976000120242436
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,1,0.015024000157912573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,2047,0.08326399823029836
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,2047,0.046341334780057274
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,3,0.016917333006858826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,3,0.015141333142916361
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,7,0.017263999829689663
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,7,0.015178666760524115
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,4095,0.15853333473205566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,31,0.01706133286158244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,31,0.01492799942692121
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,4095,0.08398933211962382
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,63,0.016976000120242436
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,15,0.016986666868130367
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,15,0.01526933287580808
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,63,0.015029333531856537
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,127,0.014853333433469137
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,255,0.026709333062171936
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,511,0.04433600107828776
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,511,0.0273333340883255
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,1023,0.08123733103275299
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,1023,0.04686399797598521
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,4095,0.29845333099365234
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,4095,0.1578933298587799
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,127,0.01722666621208191
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,3,0.006677333265542984
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,2047,0.08362666765848796
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,2047,0.1546453336874644
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,1,0.00701333334048589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,7,0.006618666773041089
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,7,0.006725333631038666
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,255,0.014725333700577417
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,31,0.006762666627764702
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,63,0.006682666639486949
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,3,0.006645333642760913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,1,0.00679466687142849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,127,0.006757333253820737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,255,0.0068693334857622785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,255,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,511,0.008858666444818178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,511,0.006805333619316419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,1023,0.00890666681031386
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,1023,0.00814933329820633
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,2047,0.008693333094318708
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,2047,0.008976000050703684
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,4095,0.008736000085870424
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,4095,0.008778666456540426
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,8191,0.012709333250919977
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,8191,0.010837333897749582
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,16383,0.014896000425020853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,16383,0.013045333325862885
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,32767,0.031061333914597828
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,32767,0.016373333831628162
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,65535,0.04994666576385498
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,65535,0.03129599988460541
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,131071,0.086709330479304
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,131071,0.05016533533732096
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,15,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,1,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,1,0.006981333096822103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,3,0.006624000146985054
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,3,0.006757333253820737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,7,0.0068853336075941724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,7,0.006618666773041089
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,15,0.006762666627764702
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,15,0.006698666761318843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,127,0.006810666372378667
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,31,0.006762666627764702
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,31,0.006821333120266597
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,63,0.0068693334857622785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,63,0.007818666597207388
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,127,0.006688000013430913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,127,0.006784000123540561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,255,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,255,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,511,0.008725333337982496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,63,0.0069759997228781385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,511,0.0069919998447100324
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,1023,0.008821333448092142
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,2047,0.008826666822036108
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,2047,0.008682666967312494
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,4095,0.010922666639089584
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,4095,0.010805333654085795
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,8191,0.015024000157912573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,8191,0.012991999586423239
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,16383,0.02940800040960312
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,16383,0.014741333822409311
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,32767,0.047509332497914634
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,32767,0.02926933268706004
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,65535,0.04780800143877665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,65535,0.0839466651280721
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,131071,0.15658666690190634
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,131071,0.08708799878756206
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,15,0.006762666627764702
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,31,0.006730666384100914
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,1023,0.008912000184257826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,1,0.016997333616018295
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,1,0.014848000059525171
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,3,0.014677333335081736
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,3,0.016970666746298473
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,7,0.017136000096797943
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,7,0.014906667172908783
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,15,0.016885332763195038
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,15,0.01504533365368843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,31,0.014741333822409311
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,31,0.017029333859682083
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,63,0.016890666137139004
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,127,0.017029333859682083
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,63,0.015130666395028433
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,127,0.01488000030318896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,255,0.026954665780067444
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,255,0.015050667027632395
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,511,0.04513599971930186
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,511,0.0272533322374026
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,1,0.027386667827765148
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,1023,0.08256533245245616
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,1023,0.04706133405367533
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,1,0.02332799881696701
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,2047,0.08370666702588399
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,2047,0.15577066938082376
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,3,0.027136000494162243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,3,0.023242667317390442
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,7,0.027445333699385326
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,15,0.023077333966890972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,7,0.023103999594847362
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,15,0.027056001126766205
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,63,0.02737066646416982
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,63,0.023024000227451324
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,127,0.029066666960716248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,127,0.023152001202106476
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,255,0.04472533365090688
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,255,0.027376001079877216
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,31,0.02719466636578242
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,31,0.023311999936898548
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,511,0.08096000055472057
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,511,0.046426668763160706
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,1023,0.15403200189272562
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,1023,0.08227733274300893
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,1,0.027722666660944622
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,1,0.02327466756105423
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,2047,0.30026666323343915
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,3,0.027130665878454845
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,2047,0.15612799922625223
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,3,0.0232640008131663
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,15,0.02739199995994568
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,7,0.023242667317390442
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,15,0.023130667706330616
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,31,0.023082666099071503
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,63,0.023157333334287006
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,127,0.02939733366171519
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,7,0.02743999908367793
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,127,0.023546665906906128
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,255,0.04571733375390371
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,31,0.027141332626342773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,255,0.02882133424282074
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,63,0.02717866748571396
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,1,0.04878933231035868
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,511,0.0461760014295578
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,1,0.039621333281199135
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,511,0.08261333405971527
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,3,0.0496373325586319
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,1023,0.08514133095741272
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,1023,0.15617066621780396
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,3,0.039434666434923805
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,15,0.04778666794300079
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,7,0.048021331429481506
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,7,0.03942399968703588
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,31,0.049423997600873314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,15,0.039594667653242745
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,63,0.03958933303753535
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,127,0.05090666810671488
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,31,0.039808000127474465
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,63,0.04855999847253164
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,127,0.041493333876132965
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,1,0.007087999954819679
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,3,0.007034666836261749
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,1,0.006624000146985054
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,3,0.006714666883150737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,7,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,7,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,255,0.08133866886297862
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,15,0.00679466687142849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,15,0.00706666645904382
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,255,0.04849066833655039
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,31,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,63,0.0068800002336502075
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,63,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,127,0.006725333631038666
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,255,0.007034666836261749
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,255,0.006751999879876773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,511,0.008901333436369896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,511,0.00706666645904382
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,1023,0.008826666822036108
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,1023,0.008672000219424566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,2047,0.008874666566650072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,2047,0.008757333581646284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,4095,0.01098666712641716
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,4095,0.010751999914646149
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,8191,0.014394666999578476
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,8191,0.012821332861979803
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,16383,0.029391999046007793
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,16383,0.014890667051076889
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,32767,0.047839999198913574
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,32767,0.02921066681543986
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,31,0.006650666395823161
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,65535,0.0855466624101003
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,65535,0.049253334601720176
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,127,0.007018666714429855
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,511,0.15608533223470053
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,1,0.007386666412154834
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,131071,0.16030399998029074
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,1,0.006981333096822103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,3,0.007184000064929326
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,3,0.006709333509206772
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,7,0.006714666883150737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,15,0.00721066693464915
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,15,0.006671999891599019
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,31,0.006671999891599019
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,31,0.006981333096822103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,63,0.006709333509206772
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,63,0.006815999746322632
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,127,0.006677333265542984
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,127,0.007050666958093643
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,255,0.007045333584149678
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,255,0.006773333375652631
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,511,0.00879466657837232
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,511,0.007061333085099856
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,1023,0.009066666786869368
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,1023,0.008613333106040955
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,2047,0.010911999891201654
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,2047,0.008805333326260248
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,4095,0.012928000340859095
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,511,0.08276266853014629
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,4095,0.010816000401973724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,8191,0.027109332382678986
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,16383,0.044997334480285645
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,32767,0.08266666531562805
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,32767,0.046154667933781944
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,131071,0.08668800195058186
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,7,0.006805333619316419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,65535,0.15546666582425436
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,65535,0.08318399886290233
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,1,0.006965333595871925
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,1,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,131071,0.30375999212265015
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,3,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,131071,0.16204800208409628
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,7,0.006677333265542984
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,3,0.007104000076651573
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,7,0.006735999758044879
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,15,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,15,0.007018666714429855
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,16383,0.027322667340437572
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,31,0.00660800002515316
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,63,0.006895999734600385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,127,0.006709333509206772
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,8191,0.013088000317414602
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,127,0.006805333619316419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,255,0.006677333265542984
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,255,0.006704000135262807
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,511,0.008933333059151968
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,511,0.007034666836261749
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,1023,0.008746666833758354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,2047,0.010794666906197866
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,2047,0.008703999842206636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,4095,0.012586666891972223
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,4095,0.010879999647537867
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,8191,0.027215999861558277
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,8191,0.013072000195582708
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,1023,0.3006719946861267
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,16383,0.04554133117198944
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,16383,0.027802666028340656
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,32767,0.0828959991534551
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,32767,0.04610666632652283
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,65535,0.1567039986451467
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,31,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,1,0.006981333096822103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,1,0.007536000261704127
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,1023,0.1572533349196116
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,3,0.00730666642387708
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,1023,0.008912000184257826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,15,0.0069866664707660675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,7,0.007151999821265538
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,15,0.007162666569153468
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,31,0.007178666690985362
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,63,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,31,0.006751999879876773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,63,0.007354666789372762
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,127,0.007162666569153468
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,63,0.006981333096822103
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,255,0.006751999879876773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,3,0.0068853336075941724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,511,0.007680000116427739
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,511,0.00730666642387708
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,1023,0.008767999708652496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,1023,0.008954666554927826
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,2047,0.009088000282645226
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,4095,0.010234666367371878
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,4095,0.009029333169261614
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,127,0.007216000308593114
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,7,0.006858666737874349
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,8191,0.009301333377758661
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,16383,0.012773333738247553
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,16383,0.012709333250919977
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,8191,0.00884799969693025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,32767,0.017162666966517765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,131071,0.16152000427246094
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,255,0.007018666714429855
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,65535,0.025600001215934753
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,65535,0.023589332898457844
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,131071,0.3062453269958496
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,131071,0.027066667874654133
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,1,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,1,0.006709333509206772
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,3,0.006821333120266597
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,131071,0.02932800104220708
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,7,0.0069866664707660675
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,7,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,3,0.00655466690659523
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,15,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,31,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,31,0.006858666737874349
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,63,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,63,0.006575999781489372
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,127,0.006741333131988843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,127,0.006746666505932808
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,255,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,255,0.006581333155433337
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,511,0.008869333192706108
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,511,0.007141333073377609
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,1023,0.008597333605090777
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,2047,0.008922666932145754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,2047,0.008634666601816813
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,2047,0.00903466654320558
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,4095,0.008639999975760778
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,4095,0.008634666601816813
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,8191,0.009008000294367472
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,8191,0.008938666433095932
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,16383,0.012885333349307379
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,16383,0.012975999464591345
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,32767,0.0174346665541331
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,32767,0.017093333105246227
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,65535,0.02160000056028366
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,65535,0.02096533278624217
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,131071,0.03856533269087473
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,131071,0.021642667551835377
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,1,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,1,0.007130666946371396
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,3,0.006842666616042455
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,3,0.0069440001000960665
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,65535,0.0844640036424001
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,7,0.006821333120266597
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,7,0.006773333375652631
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,32767,0.01711999997496605
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,15,0.007045333584149678
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,31,0.006911999856432279
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,63,0.0069973332186539965
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,63,0.006927999978264173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,127,0.006831999868154526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,15,0.006688000013430913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,127,0.006720000257094701
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,255,0.006874666859706243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,255,0.006837333242098491
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,511,0.008703999842206636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,511,0.0086666668454806
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,1023,0.00860799973209699
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,1023,0.009701333319147428
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,1023,0.008879999940594038
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,2047,0.01080000028014183
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,2047,0.010682666053374609
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,8191,0.045456002155939736
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,8191,0.027248000105222065
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,4095,0.02717866748571396
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,4095,0.013082666943470636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,32767,0.1574026644229889
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,16383,0.08282133440176646
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,16383,0.04585599899291992
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,32767,0.08408533533414204
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,15,0.006677333265542984
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,1,0.009370666618148485
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,31,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,3,0.007109333450595538
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,1,0.006805333619316419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,3,0.007226666435599327
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,7,0.007071999832987785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,7,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,15,0.007418666655818622
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,31,0.006858666737874349
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,63,0.006837333242098491
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,65535,0.30475733677546185
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,65535,0.15980266531308493
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,127,0.007040000210205714
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,255,0.007274666801095009
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,131071,0.6007039944330851
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,255,0.007055999711155891
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,511,0.008885333314538002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,131071,0.30482133229573566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,127,0.006805333619316419
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,1023,0.008576000109314919
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,1023,0.008618666479984919
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,2047,0.008506666868925095
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,511,0.00690133310854435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,8191,0.00997866690158844
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,8191,0.00854399986565113
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,2047,0.008672000219424566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,16383,0.01358933374285698
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,16383,0.012736000120639801
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,32767,0.017269333203633625
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,32767,0.01543466622630755
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,65535,0.03349333256483078
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,65535,0.01724799970785777
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,131071,0.05228800078233083
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,131071,0.034858666360378265
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,1,0.0086666668454806
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,1,0.0068853336075941724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,15,0.007477333148320516
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,3,0.008597333605090777
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,63,0.006735999758044879
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,3,0.006757333253820737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,7,0.00867733359336853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,7,0.00696000022192796
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,15,0.008320000022649765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,15,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,31,0.008645333349704742
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,63,0.008757333581646284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,63,0.007034666836261749
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,127,0.00871999996403853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,4095,0.008789333204428354
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,127,0.0069226666043202085
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,255,0.00871999996403853
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,511,0.009098666409651438
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,511,0.008965333302815756
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,1023,0.01110400011142095
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,1023,0.009162666896979014
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,2047,0.02609066665172577
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,2047,0.012693333129088083
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,31,0.00696000022192796
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,4095,0.045050665736198425
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,4095,0.02717866748571396
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,16383,0.1566933294137319
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,8191,0.045456002155939736
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,31,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,4095,0.008629333227872849
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,255,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,16383,0.08434133728345235
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,1,0.009136000027259191
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,32767,0.3040906588236491
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,1,0.009045333291093508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,3,0.009290666629870733
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,8191,0.0825386643409729
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,65535,0.6006400187810262
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,7,0.010368000095089277
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,3,0.008693333094318708
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,65535,0.3074293335278829
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,15,0.009082666908701261
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,32767,0.1567200024922689
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,31,0.00902399979531765
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,31,0.008650666723648706
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,15,0.008885333314538002
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,63,0.008853333070874214
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,127,0.008901333436369896
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,127,0.008672000219424566
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,255,0.00901333304742972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,63,0.009248000259200731
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,255,0.009002666920423508
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,511,0.012159999459981918
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,511,0.010853332777818045
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,1023,0.02665599932273229
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,1023,0.012853333105643591
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,7,0.008645333349704742
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,2047,0.045114666223526
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,2047,0.027215999861558277
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,8191,0.15662933389345804
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,8191,0.08396800359090169
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,16383,0.16031466921170553
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,16383,0.304970661799113
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,4095,0.08271466692288716
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,4095,0.04689066608746847
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,1,0.012634667257467905
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,1,0.01101333275437355
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,3,0.012517333030700684
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,3,0.010581333190202713
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,7,0.012437333663304647
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,7,0.010677333921194077
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,31,0.012527999778588613
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,31,0.010869332899649939
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,63,0.012837332983811697
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,63,0.010938666760921478
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,15,0.010832000523805618
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,15,0.012757333616415659
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,127,0.012410666793584824
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,127,0.010735999792814255
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,255,0.0120319997270902
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,255,0.010890666395425797
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,511,0.027136000494162243
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,511,0.013466666142145792
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,1023,0.045696000258127846
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,1023,0.027061333258946735
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,2047,0.08232533435026805
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,2047,0.04585599899291992
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,1,0.006751999879876773
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,7,0.007034666836261749
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,7,0.006853333363930385
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,1,0.006757333253820737
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,15,0.006634666894872983
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,8191,0.30852266152699787
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,15,0.006746666505932808
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,31,0.0069333333522081375
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,63,0.006634666894872983
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,63,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,8191,0.15920533736546835
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,127,0.007002666592597961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,127,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,511,0.008602666358153025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,511,0.007135999699433644
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,31,0.0068853336075941724
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,1023,0.008997333546479544
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,1023,0.008757333581646284
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,2047,0.009066666786869368
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,2047,0.008778666456540426
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,255,0.011221333096424738
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,4095,0.009216000015536943
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,8191,0.011503999431928
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,4095,0.008703999842206636
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,8191,0.010965333630641302
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,16383,0.015013333410024643
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,3,0.00701333334048589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,16383,0.01310933381319046
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,3,0.006789333497484525
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,32767,0.015098666151364645
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,65535,0.031130666534105938
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,4095,0.15846932927767435
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,131071,0.08659733335177104
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,131071,0.050197333097457886
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,4095,0.0846453309059143
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,255,0.00666133314371109
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,32767,0.0303413321574529
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,65535,0.04854399959246317
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,1,0.017093333105246227
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,1,0.014906667172908783
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,3,0.015066667149464289
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,7,0.014783999572197596
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,15,0.01682666689157486
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,3,0.017029333859682083
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,15,0.01481066644191742
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,7,0.017184000462293625
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,31,0.017173333714405697
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,31,0.01481066644191742
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,63,0.01720533271630605
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,63,0.014933332800865173
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,127,0.01720000058412552
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,127,0.014767999450365702
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,255,0.014783999572197596
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,255,0.02699200063943863
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,511,0.04545066754023234
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,1023,0.08197866876920064
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,1023,0.046021332343419395
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,511,0.027290667096773785
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,2047,0.15500266353289285
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,1,0.02752533306678136
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,2047,0.08337066570917766
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,1,0.023024000227451324
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,3,0.027632000545660656
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,4095,0.30114134152730304
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,4095,0.16005333264668783
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,3,0.023391999304294586
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,7,0.023152001202106476
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,7,0.02735999971628189
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,15,0.023168000082174938
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,15,0.02719466636578242
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,31,0.027690666417280834
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,63,0.02309333284695943
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,127,0.02942399928967158
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,127,0.023007998863856
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,255,0.0451200008392334
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,255,0.028965334097544353
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,31,0.023077333966890972
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,1,0.006640000268816948
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,1,0.006698666761318843
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,511,0.08248533308506012
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,3,0.00697066696981589
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,7,0.007002666592597961
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,7,0.006688000013430913
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,15,0.006954666847983996
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,15,0.006917333230376244
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,31,0.006890666360656421
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,31,0.006826666494210561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,63,0.00720000018676122
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,63,0.006671999891599019
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,127,0.006682666639486949
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,511,0.04549333453178406
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,3,0.006864000111818314
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,127,0.006560000280539195
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,255,0.007167999943097432
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,511,0.008623999853928884
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,511,0.007216000308593114
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,1023,0.008922666932145754
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,1023,0.008874666566650072
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,2047,0.00884799969693025
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,2047,0.008858666444818178
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,4095,0.010656000425418219
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,4095,0.01108266661564509
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,63,0.02773866554101308
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,8191,0.013557333499193192
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,8191,0.012661332885424295
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,16383,0.030282666285832722
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,255,0.006826666494210561
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,32767,0.047872001926104225
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,32767,0.029706666866938274
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,16383,0.015029333531856537
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,65535,0.04824000100294749
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,131071,0.15635733803113303
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,1023,0.0827893316745758
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,1023,0.1554026703039805
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,131071,0.08678399523099263
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,65535,0.08478933572769165
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,2047,0.3022080063819885
VLLM,0.19.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,2047,0.15684266885121664
