framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,1,0.008613333106040955
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,31,0.008703999842206636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,31,0.008320000022649765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,15,0.008394666636983553
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,15,0.008842666943868002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,63,0.008816000074148178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,3,0.008789333204428354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,63,0.008586666857202848
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,127,0.008746666833758354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,127,0.00874133345981439
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,255,0.008517333616813024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,7,0.00915733352303505
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,511,0.00871999996403853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,511,0.008896000062425932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,1023,0.012655999511480331
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,3,0.00878399983048439
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,1023,0.01098666712641716
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,2047,0.02717866748571396
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,4095,0.0273333340883255
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,4095,0.045456002155939736
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,1,0.008661333471536636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,1,0.010053333515922228
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,3,0.008656000097592672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,3,0.00903466654320558
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,7,0.00892800030608972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,7,0.008789333204428354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,15,0.0086666668454806
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,15,0.008869333192706108
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,31,0.009029333169261614
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,31,0.008693333094318708
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,7,0.008767999708652496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,63,0.008778666456540426
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,63,0.00897066667675972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,1,0.008799999952316284
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,127,0.008597333605090777
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,255,0.008645333349704742
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,255,0.00879466657837232
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,511,0.012890666723251343
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,511,0.01073066641887029
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,float16,255,0.008762666955590248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,1023,0.012714666624863943
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,2047,0.044879997769991554
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,2047,0.027034667630990345
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,1,0.006810666372378667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,4095,0.04576000074545542
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,3,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,1,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,4095,0.08130133152008057
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,7,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,15,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,15,0.006741333131988843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,31,0.007018666714429855
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,31,0.006709333509206772
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,63,0.006735999758044879
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,63,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,127,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,127,0.006709333509206772
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,255,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,255,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,511,0.008661333471536636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,511,0.007600000128149986
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,1023,0.008874666566650072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,1023,0.008629333227872849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,2047,0.00898133342464765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,2047,0.008613333106040955
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,4095,0.00902399979531765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,4095,0.008879999940594038
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,8191,0.010687999427318573
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,8191,0.008858666444818178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,16383,0.014709333578745524
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,127,0.008943999807039896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,1,0.006645333642760913
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,3,0.007135999699433644
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,3,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,float16,1023,0.02683199942111969
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,7,0.006528000036875407
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,7,0.006714666883150737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,15,0.006655999769767125
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,15,0.006506666541099548
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,31,0.006773333375652631
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,31,0.007242666557431221
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,2047,0.013002666334311167
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,63,0.0069759997228781385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,63,0.006735999758044879
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,7,0.007247999931375186
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,127,0.006581333155433337
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,127,0.006629333520929019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,255,0.006805333619316419
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,511,0.006720000257094701
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,511,0.008736000085870424
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,1023,0.00860799973209699
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,1023,0.008789333204428354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,2047,0.008821333448092142
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,2047,0.0085333331177632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,4095,0.008896000062425932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,4095,0.008581333483258883
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,8191,0.013007999708255133
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,16383,0.012965332716703415
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,16383,0.015157333264748255
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,1,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,3,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,16383,0.01488000030318896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,3,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,float16,1,0.00679466687142849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,7,0.006778666749596596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,15,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,31,0.006751999879876773
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,31,0.006981333096822103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,63,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,float16,3,0.007007999966541926
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,63,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,127,0.006831999868154526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,127,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,255,0.006704000135262807
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,255,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,255,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,511,0.008863999818762144
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,511,0.008661333471536636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,1023,0.008922666932145754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,1023,0.008767999708652496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,2047,0.008874666566650072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,2047,0.008639999975760778
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,4095,0.009125333279371262
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,8191,0.010944000134865442
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,4095,0.008682666967312494
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,8191,0.012890666723251343
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,8191,0.011194666226704916
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,1,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,16383,0.014783999572197596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,1,0.006725333631038666
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,1,0.006693333387374878
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,3,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,3,0.006709333509206772
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,7,0.006805333619316419
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,15,0.006778666749596596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,15,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,7,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,31,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,15,0.006762666627764702
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,31,0.006698666761318843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,63,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,63,0.006789333497484525
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,127,0.006746666505932808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,127,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,255,0.00660800002515316
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,255,0.006800000245372455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,511,0.007045333584149678
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,511,0.00874133345981439
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,1023,0.008447999755541483
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,1023,0.008890666688481966
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,2047,0.008698666468262672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,2047,0.008714666590094566
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,4095,0.010437333335479101
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,4095,0.010773333410422007
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,8191,0.01470400020480156
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,8191,0.014042666802803675
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,16383,0.015077333897352219
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,16383,0.029050665597120922
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,1,0.010693332801262537
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,1,0.008992000172535578
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,3,0.010645333677530289
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,3,0.008896000062425932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,7,0.010762666662534079
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,7,0.008762666955590248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,15,0.010575999816258749
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,15,0.00902399979531765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,31,0.010693332801262537
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,31,0.008736000085870424
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,63,0.01073066641887029
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,63,0.008954666554927826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,127,0.010687999427318573
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,127,0.00890666681031386
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,float16,16383,0.015029333531856537
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,255,0.010778666784365972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,255,0.00878399983048439
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,511,0.01101333275437355
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,2047,0.04554666578769684
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,float16,7,0.006618666773041089
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,1023,0.027050666511058807
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,2047,0.0278613343834877
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,1,0.012800000607967377
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,1,0.010629333555698395
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,3,0.010821333775917688
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,3,0.012703999876976013
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,7,0.010853332777818045
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,15,0.012885333349307379
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,7,0.012357333054145178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,15,0.01055466632048289
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,31,0.010650667051474253
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,31,0.012757333616415659
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,63,0.012383999923865
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,63,0.010773333410422007
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,127,0.010682666053374609
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,255,0.012506666282812754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,511,0.027274665733178455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,511,0.013989333063364029
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,1023,0.04565866788228353
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,1023,0.028101332485675812
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,1,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,1,0.0069919998447100324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,3,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,3,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,2047,0.04669866462548574
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,2047,0.08246399958928426
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,float16,511,0.012837332983811697
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,7,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,7,0.0069919998447100324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,15,0.007045333584149678
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,1023,0.015034666905800501
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,15,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,31,0.006800000245372455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,31,0.007034666836261749
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,63,0.006954666847983996
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,127,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,127,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,255,0.006677333265542984
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,511,0.008672000219424566
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,511,0.008752000207702318
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,1023,0.009109333157539368
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,1023,0.008746666833758354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,2047,0.008805333326260248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,4095,0.012058666596810022
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,4095,0.01073066641887029
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,8191,0.014853333433469137
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,float16,127,0.012794667234023413
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,8191,0.013258667041858038
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,16383,0.029125332832336426
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,16383,0.01687466725707054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,1,0.006655999769767125
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,1,0.006640000268816948
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,3,0.006949333474040031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,3,0.006629333520929019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,7,0.006810666372378667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,7,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,15,0.00666133314371109
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,15,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,31,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,31,0.006634666894872983
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,63,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,127,0.006730666384100914
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,127,0.006640000268816948
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,255,0.006746666505932808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,255,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,255,0.006618666773041089
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,511,0.008762666955590248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,511,0.0069440001000960665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,1023,0.00871999996403853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,float16,2047,0.008992000172535578
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,2047,0.010869332899649939
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,2047,0.00867733359336853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,4095,0.012842666357755661
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,8191,0.02719466636578242
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,8191,0.012944000462690989
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,float16,16383,0.045221333702405296
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,255,0.010762666662534079
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,16383,0.027034667630990345
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,1,0.014490666488806406
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,1,0.012944000462690989
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,3,0.012778667112191519
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,3,0.014848000059525171
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,7,0.014117332796255747
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,7,0.012810666114091873
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,63,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,15,0.014848000059525171
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,31,0.014773332824309668
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,31,0.012815999488035837
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,63,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,63,0.014655999839305878
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,63,0.012928000340859095
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,1023,0.00874133345981439
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,127,0.015013333410024643
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,4095,0.010794666906197866
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,255,0.014666666587193808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,255,0.012671999633312225
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,511,0.028181334336598713
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,511,0.01651200031240781
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,1,0.014671999961137772
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,1,0.016842667013406754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,float16,1023,0.046154667933781944
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,1023,0.029135999580224354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,3,0.01714666684468587
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,3,0.015050667027632395
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,7,0.014741333822409311
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,7,0.016901332885026932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,15,0.012986666212479273
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,31,0.014864000181357065
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,31,0.016901332885026932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,63,0.014949332922697067
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,63,0.01693333312869072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,15,0.014736000448465347
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,15,0.01695466662446658
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,127,0.017071999609470367
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,127,0.014837333311637243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,127,0.012944000462690989
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,255,0.015077333897352219
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,1023,0.08190933366616567
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,1023,0.046538665890693665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,511,0.0458133320013682
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,511,0.027232001225153606
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,1,0.022634667654832203
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,3,0.023034666975339253
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,3,0.016917333006858826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,7,0.021744000415007275
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,7,0.01708799973130226
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,15,0.021781332790851593
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,15,0.01711999997496605
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,31,0.02123733361562093
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,31,0.01714666684468587
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,63,0.02139200021823247
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,63,0.01721599946419398
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,127,0.021920000513394673
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,127,0.016938666502634685
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,float16,255,0.027450665831565857
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,1,0.01729600007335345
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,float16,255,0.027066667874654133
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,1,0.027535999814669292
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,255,0.01714133347074191
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,1,0.023247999449570973
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,3,0.027221334477265675
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,7,0.02717333287000656
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,3,0.023189333577950794
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,7,0.023365333676338196
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,15,0.023269332945346832
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,31,0.027349332968393963
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,31,0.023007998863856
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,63,0.027429332335789997
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,127,0.023071999351183575
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,255,0.02903466671705246
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,1,0.03730666637420654
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,1,0.027210667729377747
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,3,0.03764266769091288
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,7,0.03759466608365377
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,7,0.027109332382678986
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,15,0.03745066622893015
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,15,0.027189334233601887
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,15,0.02735466758410136
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,63,0.022986667851607006
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,31,0.037248000502586365
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,127,0.029418667157491047
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,31,0.02717866748571396
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,63,0.03736000011364619
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,float16,255,0.045978665351867676
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,63,0.027269333600997925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,float16,127,0.03763733307520548
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,127,0.027136000494162243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,3,0.02741333345572154
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,1,0.04964800179004669
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,1,0.039706667264302574
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,3,0.049360002080599465
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,3,0.03976000100374222
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,7,0.04894400139649709
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,7,0.03944533318281174
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,15,0.04929600159327189
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,15,0.0393653338154157
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,31,0.04884799818197886
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,31,0.039850667119026184
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,1,0.006842666616042455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,3,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,7,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,63,0.048783997694651283
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,7,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,15,0.006954666847983996
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,31,0.006725333631038666
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,31,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,15,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,63,0.007071999832987785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,63,0.03967999915281931
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,63,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,127,0.00702400008837382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,127,0.006720000257094701
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,255,0.006981333096822103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,511,0.009093333035707474
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,float16,127,0.05159999926884969
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,127,0.04040000090996424
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,255,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,1023,0.008933333059151968
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,1023,0.008879999940594038
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,2047,0.008933333059151968
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,4095,0.01090666651725769
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,8191,0.02734400083621343
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,8191,0.014805333067973455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,16383,0.04549333453178406
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,16383,0.028101332485675812
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,2047,0.010650667051474253
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,4095,0.012831999609867731
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,1,0.006624000146985054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,3,0.006645333642760913
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,3,0.006954666847983996
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,7,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,7,0.006714666883150737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,15,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,float16,1,0.007098666702707608
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,31,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,31,0.006666666517655055
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,63,0.0069759997228781385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,63,0.006650666395823161
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,3,0.006762666627764702
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,127,0.0069759997228781385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,255,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,255,0.007007999966541926
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,511,0.007578666632374127
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,511,0.008842666943868002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,1023,0.009045333291093508
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,1023,0.008752000207702318
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,2047,0.011183999478816986
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,2047,0.010735999792814255
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,4095,0.02698666602373123
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,511,0.0085333331177632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,4095,0.012928000340859095
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,8191,0.045040001471837364
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,8191,0.02699200063943863
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,16383,0.04543466866016388
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,16383,0.08065066734949748
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,float16,1,0.006698666761318843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,15,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,127,0.006613333399097125
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,float16,1,0.06818133095900218
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,1,0.047584002216657005
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,3,0.04756266872088114
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,float16,3,0.06803733110427856
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,float16,7,0.06824533144632976
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,7,0.04753600060939789
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,float16,15,0.068271999557813
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,15,0.04747200012207031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,float16,31,0.06811733543872833
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,31,0.04720533390839895
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,float16,1,0.0888159970442454
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,float16,63,0.06817066669464111
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,63,0.047237331668535866
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,1,0.07241066793600719
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,float16,3,0.08989333113034566
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,3,0.07231466472148895
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,float16,7,0.08940266569455464
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,7,0.0731573353211085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,float16,15,0.08875200152397156
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,15,0.07392533123493195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,float16,31,0.0897866686185201
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,float16,63,0.08948266506195068
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,63,0.07275199890136719
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,31,0.07229866584142049
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,float16,1,0.12784533699353537
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,1,0.08683733145395915
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,float16,3,0.12961066762606302
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,3,0.08821866909662883
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,float16,7,0.12931733330090842
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,7,0.08866133292516072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,float16,15,0.12939733266830444
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,15,0.08799999952316284
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,float16,31,0.12782399853070578
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,float16,1,0.16854933897654215
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,1,0.13800533612569174
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,3,0.1378613313039144
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,float16,3,0.1693120002746582
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,1,0.008943999807039896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,31,0.0867733359336853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,1,0.007093333328763644
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,float16,7,0.16849066813786825
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,3,0.007743999982873599
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,3,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,7,0.008736000085870424
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,7,0.0069919998447100324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,7,0.1379093329111735
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,15,0.00871999996403853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,15,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,31,0.00702400008837382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,31,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,63,0.008885333314538002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,127,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,63,0.006784000123540561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,127,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,255,0.008943999807039896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,255,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,511,0.008693333094318708
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,511,0.008943999807039896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,1023,0.01099733387430509
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,1023,0.009018666421373686
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,2047,0.012565333396196365
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,2047,0.010911999891201654
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,4095,0.027322667340437572
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,float16,8191,0.045408000548680626
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,8191,0.029077333708604176
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,1,0.008581333483258883
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,3,0.008623999853928884
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,3,0.00701333334048589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,7,0.008661333471536636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,7,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,float16,15,0.16808533668518066
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,15,0.0069440001000960665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,31,0.00874133345981439
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,31,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,63,0.008885333314538002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,float16,31,0.16865599155426025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,127,0.008693333094318708
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,255,0.008863999818762144
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,4095,0.014885333677132925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,127,0.007055999711155891
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,255,0.007093333328763644
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,1,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,511,0.008832000195980072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,511,0.008682666967312494
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,31,0.1378506620724996
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,1023,0.011061333119869232
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,1023,0.009045333291093508
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,2047,0.02632533262173335
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,2047,0.013023999830087027
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,4095,0.04362666606903076
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,15,0.008570666735370954
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,float16,8191,0.08063466846942902
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,8191,0.045066664616266884
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,1,0.008565333361426989
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,4095,0.027024000883102417
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,1,0.007786666974425316
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,3,0.008703999842206636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,3,0.007167999943097432
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,7,0.00855466661353906
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,7,0.008767999708652496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,15,0.0086666668454806
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,15,0.008767999708652496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,31,0.008997333546479544
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,31,0.0069759997228781385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,63,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,63,0.007221333061655362
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,127,0.008661333471536636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,127,0.007151999821265538
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,255,0.008597333605090777
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,255,0.007829333345095316
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,511,0.008639999975760778
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,511,0.008837333569924036
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,1023,0.011168000598748526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,1023,0.010698666175206503
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,2047,0.012986666212479273
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,4095,0.04447466631730398
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,1,0.00879466657837232
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,4095,0.02717333287000656
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,1,0.008943999807039896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,3,0.00914666677514712
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,3,0.008672000219424566
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,7,0.008538666491707167
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,15,0.009018666421373686
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,15,0.008842666943868002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,31,0.009050666665037474
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,31,0.00874133345981439
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,63,0.008762666955590248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,63,0.008896000062425932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,127,0.00873066671192646
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,127,0.0085333331177632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,255,0.008965333302815756
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,255,0.009018666421373686
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,511,0.012416000167528788
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,511,0.010794666906197866
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,1023,0.02630399912595749
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,2047,0.027295999228954315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,2047,0.04460266729195913
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,2047,0.027306665976842243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,4095,0.08051200211048126
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,1,0.006773333375652631
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,1,0.00701333334048589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,4095,0.04574933151404063
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,3,0.007120000198483467
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,3,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,7,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,7,0.006949333474040031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,15,0.007029333462317784
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,15,0.006597333277265231
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,31,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,31,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,63,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,63,0.007802666475375493
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,127,0.007429333403706551
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,float16,7,0.00903466654320558
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,127,0.006949333474040031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,255,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,255,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,511,0.008826666822036108
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,511,0.009173333023985228
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,1023,0.009002666920423508
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,1023,0.009002666920423508
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,2047,0.008949333180983862
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,2047,0.008789333204428354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,15,0.13768000404040018
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,4095,0.009018666421373686
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,8191,0.010954666882753372
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,8191,0.009877333417534828
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,16383,0.015066667149464289
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,16383,0.014117332796255747
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,1,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,1,0.00702400008837382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,3,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,7,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,7,0.006746666505932808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,15,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,3,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,31,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,31,0.007135999699433644
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,float16,63,0.008672000219424566
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,63,0.0069919998447100324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,127,0.006757333253820737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,127,0.006618666773041089
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,1023,0.012869333227475485
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,255,0.006602666651209195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,255,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,511,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,511,0.00877333308259646
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,1023,0.00874133345981439
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,1023,0.008570666735370954
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,2047,0.008901333436369896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,2047,0.008570666735370954
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,4095,0.008949333180983862
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,4095,0.008576000109314919
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,8191,0.01081066702802976
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,8191,0.012831999609867731
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,16383,0.013114667187134424
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,float16,16383,0.014848000059525171
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,1,0.006954666847983996
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,1,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,3,0.0068800002336502075
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,3,0.006688000013430913
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,7,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,7,0.006522666662931442
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,15,0.006751999879876773
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,15,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,31,0.007018666714429855
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,31,0.006528000036875407
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,63,0.006735999758044879
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,127,0.007050666958093643
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,127,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,255,0.006682666639486949
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,255,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,511,0.008682666967312494
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,511,0.00879466657837232
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,1023,0.00879466657837232
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,2047,0.008879999940594038
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,2047,0.008805333326260248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,15,0.006634666894872983
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,4095,0.008698666468262672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,8191,0.012810666114091873
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,63,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,8191,0.012853333105643591
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,16383,0.01498666654030482
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,16383,0.014794666320085526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,1,0.006704000135262807
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,1,0.006640000268816948
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,3,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,3,0.006720000257094701
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,7,0.006751999879876773
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,7,0.00655466690659523
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,15,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,15,0.006784000123540561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,31,0.0069919998447100324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,63,0.006714666883150737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,63,0.006666666517655055
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,127,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,127,0.006831999868154526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,255,0.006810666372378667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,63,0.007002666592597961
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,511,0.008645333349704742
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,511,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,1023,0.008805333326260248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,1023,0.008752000207702318
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,2047,0.008703999842206636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,1023,0.00871999996403853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,4095,0.010890666395425797
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,4095,0.010725333044926325
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,float16,4095,0.009008000294367472
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,8191,0.014522666732470194
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,8191,0.012666666259368261
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,float16,16383,0.02908266584078471
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,16383,0.014885333677132925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,1,0.010885333021481832
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,1,0.008965333302815756
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,3,0.010751999914646149
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,3,0.008965333302815756
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,7,0.010757333288590113
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,7,0.009119999905427298
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,15,0.010549332946538925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,15,0.00898133342464765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,31,0.010751999914646149
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,31,0.00914666677514712
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,31,0.006629333520929019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,63,0.010768000036478043
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,63,0.009018666421373686
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,127,0.010661333799362183
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,127,0.008986666798591614
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,255,0.010618666807810465
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,255,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,511,0.01268799975514412
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,511,0.010853332777818045
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,2047,0.04542933404445648
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,float16,1023,0.02731200059254964
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,2047,0.008693333094318708
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,2047,0.029120000700155895
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,1,0.012784000486135483
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,1,0.010608000059922537
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,3,0.01081066702802976
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,3,0.012794667234023413
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,7,0.010773333410422007
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,7,0.012821332861979803
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,15,0.012853333105643591
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,15,0.010735999792814255
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,31,0.012693333129088083
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,31,0.010698666175206503
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,63,0.012698666503032049
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,63,0.0107893335322539
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,127,0.012773333738247553
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,127,0.010741333166758219
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,255,0.012608000387748083
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,255,0.010768000036478043
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,511,0.012917333592971167
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,255,0.008943999807039896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,1023,0.027424000203609467
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,float16,4095,0.009056000038981438
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,1,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,1,0.0069866664707660675
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,1023,0.014965333044528961
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,2047,0.04608533283074697
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,3,0.0068800002336502075
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,3,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,2047,0.0823359986146291
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,7,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,15,0.007098666702707608
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,7,0.0069759997228781385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,31,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,31,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,63,0.00707733320693175
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,63,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,127,0.006768000001708667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,127,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,255,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,255,0.006800000245372455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,511,0.008890666688481966
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,511,0.008586666857202848
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,1023,0.009103999783595404
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,1023,0.009002666920423508
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,2047,0.008890666688481966
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,2047,0.008693333094318708
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,4095,0.012069333344697952
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,4095,0.011109333485364914
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,8191,0.014720000326633453
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,8191,0.013248000293970108
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,float16,16383,0.029445332785447437
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,16383,0.01692266638080279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,1,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,1,0.006784000123540561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,3,0.006671999891599019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,7,0.006597333277265231
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,7,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,15,0.006671999891599019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,15,0.006613333399097125
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,31,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,511,0.027082666754722595
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,31,0.006634666894872983
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,63,0.007061333085099856
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,float16,1023,0.04563199977080027
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,127,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,255,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,255,0.006810666372378667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,127,0.006629333520929019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,511,0.008693333094318708
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,511,0.007242666557431221
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,1023,0.009050666665037474
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,1023,0.00855466661353906
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,2047,0.010768000036478043
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,2047,0.008746666833758354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,4095,0.012634667257467905
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,15,0.007141333073377609
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,4095,0.010725333044926325
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,8191,0.026885333160559338
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,8191,0.013077333569526672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,float16,16383,0.04445866743723551
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,16383,0.0272533322374026
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,1,0.012815999488035837
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,3,0.014864000181357065
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,3,0.012709333250919977
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,7,0.012608000387748083
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,15,0.01470400020480156
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,15,0.012736000120639801
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,3,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,31,0.014890667051076889
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,31,0.012773333738247553
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,63,0.014858666807413101
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,63,0.012703999876976013
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,127,0.012714666624863943
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,63,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,255,0.01471466695268949
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,255,0.012709333250919977
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,511,0.027130665878454845
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,1,0.016794666647911072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,1023,0.04563733438650767
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,1,0.014869333555301031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,1,0.014933332800865173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,7,0.014831999937693277
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,3,0.016879999389251072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,7,0.017210666090250015
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,3,0.015034666905800501
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,7,0.01479999969402949
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,15,0.01701333373785019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,15,0.014922666052977243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,float16,127,0.014741333822409311
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,31,0.014853333433469137
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,31,0.017152000218629837
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,63,0.01708799973130226
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,63,0.014682666709025701
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,127,0.016879999389251072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,127,0.014709333578745524
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,511,0.01621866722901662
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,255,0.014752000570297241
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,1023,0.02922133356332779
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,511,0.045466666420300804
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,511,0.027215999861558277
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,1023,0.08076266447703044
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,1023,0.046725332736968994
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,1,0.02233600119749705
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,1,0.016895999511082966
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,7,0.021903999149799347
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,7,0.01729600007335345
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,3,0.023034666975339253
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,3,0.016970666746298473
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,15,0.021274665991465252
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,15,0.01684800038735072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,31,0.02128533273935318
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,float16,255,0.027119999130566914
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,31,0.0169813334941864
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,63,0.021498667697111767
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,63,0.016895999511082966
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,127,0.02161066730817159
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,127,0.017055999487638474
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,float16,255,0.0273333340883255
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,255,0.017162666966517765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,1,0.027136000494162243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,1,0.023045333723227184
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,3,0.0271573339899381
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,3,0.023082666099071503
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,7,0.027402666707833607
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,7,0.023226665953795116
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,15,0.027349332968393963
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,31,0.02739199995994568
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,15,0.02334933231274287
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,63,0.027349332968393963
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,63,0.022991999983787537
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,127,0.02938133229811986
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,127,0.023269332945346832
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,255,0.027717334528764088
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,float16,255,0.045696000258127846
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,1,0.03732266773780187
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,1,0.027072000006834667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,3,0.03770666569471359
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,3,0.02734400083621343
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,7,0.037808001041412354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,15,0.027189334233601887
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,31,0.0372533326347669
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,15,0.03759466608365377
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,7,0.027552001178264618
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,31,0.0271573339899381
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,63,0.03726933399836222
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,float16,127,0.0383093332250913
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,1,0.039647998909155525
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,127,0.027114666998386383
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,31,0.0229120006163915
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,3,0.04964800179004669
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,3,0.039674667020638786
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,7,0.04958933095137278
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,7,0.039434666434923805
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,15,0.03928533444801966
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,15,0.0496319979429245
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,31,0.04922133187452952
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,63,0.02737066646416982
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,1,0.007045333584149678
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,31,0.03926933308442434
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,1,0.048901334404945374
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,3,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,1,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,3,0.006800000245372455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,7,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,15,0.007194666812817256
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,15,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,31,0.007098666702707608
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,31,0.007109333450595538
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,63,0.006954666847983996
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,63,0.03941333293914795
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,63,0.049072002371152244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,127,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,127,0.006720000257094701
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,127,0.04140799989302953
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,float16,127,0.05165866514046987
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,511,0.0084906667470932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,511,0.009072000160813332
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,255,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,255,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,2047,0.010709332923094431
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,2047,0.009370666618148485
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,4095,0.012842666357755661
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,8191,0.02703999976317088
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,16383,0.04566933214664459
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,1023,0.008799999952316284
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,16383,0.02731200059254964
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,1,0.006581333155433337
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,1,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,3,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,3,0.006757333253820737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,7,0.006709333509206772
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,7,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,15,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,15,0.006602666651209195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,31,0.006650666395823161
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,63,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,63,0.006746666505932808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,63,0.006725333631038666
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,127,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,127,0.006800000245372455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,255,0.006800000245372455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,255,0.006735999758044879
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,4095,0.010954666882753372
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,511,0.008810666700204214
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,511,0.007680000116427739
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,1023,0.00972800018886725
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,1023,0.00997866690158844
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,1023,0.0086666668454806
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,2047,0.011920000116030375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,2047,0.010714666297038397
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,4095,0.012784000486135483
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,4095,0.027087998886903126
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,8191,0.027024000883102417
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,16383,0.04574400186538696
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,16383,0.0804799993832906
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,float16,7,0.006789333497484525
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,31,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,8191,0.015226667126019796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,float16,8191,0.044405331214269005
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,float16,1,0.06849599877993266
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,1,0.04756799836953481
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,float16,3,0.06990399956703186
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,3,0.04666133224964142
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,7,0.046581332882245384
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,float16,7,0.06843733290831248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,float16,15,0.06896000107129414
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,15,0.047295997540156044
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,31,0.04711999992529551
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,float16,31,0.06805866460005443
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,float16,63,0.06860800087451935
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,63,0.047055999437967934
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,float16,1,0.08859200278917949
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,1,0.0721973329782486
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,float16,3,0.0888266662756602
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,3,0.0728000005086263
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,float16,7,0.08865599830945332
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,7,0.0735093355178833
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,float16,15,0.08916266759236653
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,float16,31,0.08948799967765808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,31,0.07276799778143565
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,15,0.07417599856853485
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,63,0.07226133346557617
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,float16,63,0.08936533331871033
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,float16,1,0.12961600224177042
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,1,0.086517333984375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,float16,3,0.13158933321634927
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,3,0.08707200487454732
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,float16,7,0.12891200184822083
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,7,0.08675199747085571
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,float16,15,0.1308746635913849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,15,0.08724266290664673
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,float16,31,0.12940266728401184
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,31,0.085999995470047
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,1,0.13781332969665527
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,float16,3,0.16679465770721436
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,float16,1,0.16894932587941489
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,3,0.1376106639703115
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,1,0.007125333572427432
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,1,0.006949333474040031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,3,0.006981333096822103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,3,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,7,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,float16,7,0.16765334208806357
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,7,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,15,0.0069919998447100324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,15,0.006751999879876773
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,31,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,63,0.007018666714429855
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,63,0.0069440001000960665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,127,0.008021333565314611
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,127,0.007034666836261749
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,7,0.1381386617819468
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,255,0.007114666824539502
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,255,0.006725333631038666
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,511,0.008762666955590248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,511,0.00903466654320558
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,1023,0.01071999967098236
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,1023,0.00895999992887179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,2047,0.012810666114091873
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,2047,0.010741333166758219
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,float16,15,0.1673706571261088
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,4095,0.027456000447273254
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,4095,0.01526933287580808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,15,0.13770133256912231
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,8191,0.028575999041398365
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,1,0.007104000076651573
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,float16,31,0.16865599155426025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,1,0.008277333031098047
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,3,0.007125333572427432
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,7,0.008634666601816813
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,3,0.007930666829148928
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,7,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,15,0.008592000231146812
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,31,0.008447999755541483
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,31,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,31,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,15,0.006981333096822103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,63,0.008559999987483025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,63,0.007018666714429855
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,127,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,127,0.008698666468262672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,31,0.13762666781743368
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,255,0.007034666836261749
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,255,0.008522666369875273
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,511,0.00884799969693025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,511,0.008709333216150602
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,1023,0.010832000523805618
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,1023,0.008901333436369896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,2047,0.02589333305756251
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,4095,0.026309333741664886
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,1,0.008559999987483025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,8191,0.08044800162315369
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,8191,0.04450666904449463
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,2047,0.012671999633312225
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,1,0.007018666714429855
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,3,0.008703999842206636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,7,0.00867733359336853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,3,0.007247999931375186
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,float16,8191,0.04549333453178406
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,15,0.008682666967312494
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,31,0.008816000074148178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,31,0.007242666557431221
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,63,0.008746666833758354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,63,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,127,0.008602666358153025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,127,0.008832000195980072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,255,0.008725333337982496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,255,0.007216000308593114
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,511,0.008933333059151968
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,511,0.008933333059151968
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,1023,0.011002667248249054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,1023,0.01071999967098236
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,2047,0.026650667190551758
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,2047,0.013936000565687815
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,float16,4095,0.04543466866016388
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,1,0.00898133342464765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,3,0.009088000282645226
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,3,0.008757333581646284
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,7,0.009232000137368837
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,7,0.008517333616813024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,float16,4095,0.04437333345413208
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,15,0.009018666421373686
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,15,0.008565333361426989
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,31,0.008986666798591614
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,31,0.008896000062425932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,63,0.00903466654320558
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,7,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,15,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,127,0.009039999917149544
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,127,0.00895999992887179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,255,0.009114666531483332
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,255,0.0086666668454806
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,511,0.010965333630641302
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,511,0.010954666882753372
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,1023,0.025834667185942333
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,1023,0.01310933381319046
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,2047,0.04494933287302653
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,1,0.006831999868154526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,4095,0.0458186666170756
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,2047,0.02735466758410136
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,3,0.007029333462317784
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,float16,4095,0.08062399923801422
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,3,0.007002666592597961
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,7,0.0069440001000960665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,7,0.006709333509206772
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,15,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,31,0.0069440001000960665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,4095,0.027077332139015198
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,1,0.008896000062425932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,127,0.006842666616042455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,127,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,255,0.006821333120266597
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,511,0.008656000097592672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,511,0.008517333616813024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,63,0.008997333546479544
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,1023,0.00890666681031386
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,1023,0.008581333483258883
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,2047,0.008767999708652496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,2047,0.008922666932145754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,4095,0.00903466654320558
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,4095,0.008682666967312494
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,8191,0.011029332876205444
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,8191,0.009712000067035357
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,16383,0.015024000157912573
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,16383,0.014848000059525171
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,1,0.007045333584149678
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,1,0.006629333520929019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,15,0.006954666847983996
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,3,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,3,0.006981333096822103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,float16,63,0.0069919998447100324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,7,0.006954666847983996
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,7,0.007584000006318092
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,15,0.0068800002336502075
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,15,0.006842666616042455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,31,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,31,0.00655466690659523
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,63,0.006698666761318843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,63,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,127,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,127,0.006666666517655055
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,255,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,255,0.006698666761318843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,511,0.008517333616813024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,511,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,1023,0.008698666468262672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,1023,0.008613333106040955
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,2047,0.008949333180983862
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,2047,0.00871999996403853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,4095,0.008986666798591614
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,4095,0.008949333180983862
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,8191,0.012655999511480331
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,8191,0.011007999380429586
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,float16,16383,0.015189333508412043
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,1,0.006831999868154526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,1,0.0069919998447100324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,1,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,3,0.006954666847983996
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,3,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,7,0.00726400005320708
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,7,0.006949333474040031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,15,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,31,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,63,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,63,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,127,0.00679466687142849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,127,0.006778666749596596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,255,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,255,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,511,0.00860799973209699
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,511,0.008746666833758354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,1023,0.008853333070874214
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,2047,0.008821333448092142
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,2047,0.008896000062425932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,4095,0.008992000172535578
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,4095,0.008709333216150602
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,16383,0.012837332983811697
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,8191,0.012928000340859095
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,8191,0.012837332983811697
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,16383,0.015098666151364645
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,16383,0.014730667074521383
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,15,0.00679466687142849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,1,0.007141333073377609
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,31,0.006671999891599019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,31,0.006949333474040031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,1,0.006800000245372455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,3,0.007002666592597961
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,63,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,7,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,15,0.00660800002515316
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,15,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,255,0.007120000198483467
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,31,0.006725333631038666
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,31,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,63,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,63,0.00666133314371109
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,127,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,127,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,float16,1023,0.008885333314538002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,255,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,255,0.00702400008837382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,511,0.008538666491707167
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,511,0.0069759997228781385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,1023,0.008474666625261307
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,1023,0.008885333314538002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,2047,0.008538666491707167
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,2047,0.00867733359336853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,4095,0.010741333166758219
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,4095,0.01081066702802976
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,8191,0.013072000195582708
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,8191,0.014757333944241205
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,16383,0.01482133318980535
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,float16,16383,0.029333333174387615
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,1,0.010608000059922537
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,1,0.008821333448092142
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,3,0.010981333752473196
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,7,0.008965333302815756
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,15,0.010725333044926325
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,15,0.008954666554927826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,31,0.010933333386977514
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,31,0.008976000050703684
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,63,0.010687999427318573
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,63,0.009056000038981438
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,127,0.011018666128317514
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,127,0.009050666665037474
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,255,0.010762666662534079
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,255,0.009002666920423508
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,7,0.006693333387374878
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,511,0.012986666212479273
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,511,0.011109333485364914
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,1023,0.027248000105222065
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,1023,0.014864000181357065
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,2047,0.028704000016053517
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,1,0.012762666990359625
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,1,0.010757333288590113
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,3,0.01101333275437355
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,3,0.012784000486135483
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,7,0.012784000486135483
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,7,0.010666667173306147
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,15,0.012383999923865
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,15,0.010832000523805618
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,31,0.011968000481526056
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,31,0.010858666151762009
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,63,0.012736000120639801
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,63,0.010746666540702185
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,3,0.006810666372378667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,127,0.01268799975514412
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,127,0.010944000134865442
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,255,0.012442667037248611
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,255,0.01119999960064888
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,2047,0.04582933088143667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,511,0.027285332481066387
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,511,0.013525333255529404
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,1023,0.04543999830881754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,float16,2047,0.08227733274300893
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,1023,0.027930667002995808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,1,0.006741333131988843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,3,0.007087999954819679
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,1,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,3,0.008992000172535578
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,2047,0.04606399933497111
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,7,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,15,0.006640000268816948
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,31,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,63,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,31,0.006773333375652631
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,63,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,127,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,255,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,511,0.008559999987483025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,511,0.008896000062425932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,1023,0.008858666444818178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,1023,0.008885333314538002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,2047,0.008853333070874214
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,2047,0.008933333059151968
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,4095,0.011018666128317514
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,8191,0.015040000279744467
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,8191,0.01331199953953425
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,16383,0.029904000461101532
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,16383,0.01720000058412552
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,3,0.007055999711155891
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,float16,7,0.010735999792814255
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,15,0.007040000210205714
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,1,0.006762666627764702
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,1,0.007141333073377609
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,3,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,3,0.006671999891599019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,7,0.007018666714429855
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,7,0.007061333085099856
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,127,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,15,0.006821333120266597
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,15,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,31,0.0069866664707660675
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,31,0.0068800002336502075
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,63,0.006688000013430913
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,63,0.0069759997228781385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,127,0.006655999769767125
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,127,0.006618666773041089
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,255,0.007157333195209503
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,511,0.008725333337982496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,1023,0.009066666786869368
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,4095,0.011802667131026586
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,2047,0.008842666943868002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,4095,0.012703999876976013
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,4095,0.010885333021481832
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,8191,0.027242665489514668
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,8191,0.013173333058754602
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,16383,0.02714666724205017
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,float16,7,0.0069440001000960665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,1,0.014906667172908783
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,255,0.006981333096822103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,1,0.011706666400035223
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,3,0.014896000425020853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,255,0.006528000036875407
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,3,0.01268799975514412
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,7,0.014485333114862442
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,7,0.012784000486135483
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,2047,0.01073066641887029
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,15,0.014949332922697067
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,15,0.012826666235923767
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,31,0.014757333944241205
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,31,0.012821332861979803
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,63,0.014848000059525171
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,63,0.012698666503032049
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,float16,16383,0.04477333525816599
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,127,0.014869333555301031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,127,0.012703999876976013
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,255,0.014864000181357065
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,255,0.01240533341964086
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,511,0.027285332481066387
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,511,0.0161920003592968
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,1,0.017258666455745697
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,1,0.014965333044528961
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,float16,1023,0.04615999758243561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,1023,0.029146666328112285
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,3,0.016869333883126576
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,3,0.014842666685581207
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,1023,0.008858666444818178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,7,0.016842667013406754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,15,0.01470400020480156
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,31,0.016997333616018295
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,7,0.015082667271296183
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,31,0.01523200049996376
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,15,0.016741332908471424
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,63,0.017221332838137943
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,63,0.014896000425020853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,127,0.017263999829689663
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,127,0.014778666198253632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,255,0.01482133318980535
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,255,0.02701333413521449
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,511,0.045466666420300804
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,511,0.027093333502610523
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,1023,0.04613866905371348
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,float16,1023,0.08103999992211659
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,511,0.006842666616042455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,1,0.02271999915440877
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,7,0.021274665991465252
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,7,0.0169813334941864
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,15,0.022384000321229298
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,3,0.02165866643190384
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,3,0.01716800034046173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,15,0.01682666689157486
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,31,0.021562665700912476
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,31,0.016864000509182613
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,63,0.022463999688625336
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,63,0.017071999609470367
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,127,0.02163200080394745
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,float16,255,0.027701333165168762
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,255,0.01714666684468587
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,1,0.027509334186712902
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,1,0.02332799881696701
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,3,0.027263998985290527
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,3,0.023007998863856
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,1,0.017103999853134155
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,7,0.027232001225153606
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,15,0.027215999861558277
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,7,0.023189333577950794
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,31,0.027386667827765148
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,31,0.023034666975339253
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,63,0.02735999971628189
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,63,0.023018665611743927
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,127,0.017029333859682083
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,127,0.02938666691382726
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,float16,255,0.04550399879614512
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,255,0.02869333326816559
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,1,0.02722666660944621
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,1,0.03745066622893015
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,15,0.02334933231274287
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,3,0.03758399933576584
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,3,0.02754133443037669
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,7,0.03862400104602178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,7,0.027061333258946735
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,127,0.023189333577950794
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,15,0.037461332976818085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,15,0.02739199995994568
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,31,0.027248000105222065
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,31,0.037471999724706016
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,63,0.03734933336575826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,63,0.027034667630990345
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,127,0.02735999971628189
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,float16,127,0.037445334096749626
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,1,0.04952000081539154
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,3,0.04952000081539154
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,3,0.03953066716591517
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,1,0.040261333187421165
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,7,0.04962133367856344
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,7,0.03948266555865606
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,15,0.048058668772379555
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,15,0.03975466638803482
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,31,0.0489279975493749
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,1,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,31,0.039333333571751915
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,3,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,3,0.0069973332186539965
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,63,0.04902400076389313
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,7,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,63,0.039621333281199135
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,15,0.0069440001000960665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,31,0.0069919998447100324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,63,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,15,0.00679466687142849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,63,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,31,0.006842666616042455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,127,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,127,0.006720000257094701
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,255,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,float16,127,0.050767997900644936
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,511,0.00871999996403853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,511,0.008842666943868002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,1023,0.008965333302815756
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,127,0.04063999901215235
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,2047,0.01081066702802976
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,2047,0.008885333314538002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,4095,0.012858666479587555
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,4095,0.011146667102972666
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,1023,0.00890666681031386
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,8191,0.027119999130566914
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,8191,0.014991999914248785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,16383,0.045423999428749084
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,16383,0.028160000840822857
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,1,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,1,0.006784000123540561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,3,0.006800000245372455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,3,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,7,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,15,0.006629333520929019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,15,0.006800000245372455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,31,0.00706666645904382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,1,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,31,0.006682666639486949
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,63,0.006751999879876773
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,63,0.006709333509206772
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,127,0.006714666883150737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,127,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,255,0.00666133314371109
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,255,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,511,0.008757333581646284
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,511,0.008581333483258883
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,1023,0.008698666468262672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,1023,0.009136000027259191
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,2047,0.010847999403874079
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,2047,0.011066666493813196
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,4095,0.027077332139015198
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,255,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,4095,0.012773333738247553
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,8191,0.027087998886903126
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,8191,0.044821331898371376
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,16383,0.04587733248869578
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,16383,0.08060800035794576
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,float16,7,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,float16,7,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,1,0.04610133171081543
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,float16,1,0.06805866460005443
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,float16,3,0.06846933563550313
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,float16,7,0.06831466654936473
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,3,0.04748799900213877
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,7,0.04577599962552389
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,float16,15,0.06832000116507213
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,15,0.04780800143877665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,float16,31,0.06828799843788147
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,31,0.047024001677831016
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,float16,63,0.06829866766929626
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,63,0.046442667643229164
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,float16,1,0.08771199981371562
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,1,0.07252799967924754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,float16,3,0.08876799543698628
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,3,0.07426666716734569
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,float16,7,0.08885332942008972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,7,0.07390399773915608
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,float16,15,0.08937066793441772
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,15,0.07390399773915608
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,31,0.07351999978224437
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,float16,31,0.0888853371143341
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,float16,63,0.08875200152397156
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,63,0.07240533332029979
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,float16,1,0.12929600477218628
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,1,0.08664533495903015
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,float16,3,0.12986133495966592
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,3,0.08733866612116496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,float16,7,0.1311199963092804
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,float16,15,0.13034666577974954
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,7,0.08735466996828715
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,15,0.08753066261609395
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,float16,31,0.129120002190272
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,float16,1,0.16742932796478271
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,1,0.1379039982954661
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,31,0.08678932984670003
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,float16,3,0.16828266779581705
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,1,0.007344000041484833
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,3,0.13798399766286215
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,1,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,3,0.006725333631038666
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,3,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,7,0.13798933227856955
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,7,0.00679466687142849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,7,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,float16,7,0.16741865873336792
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,15,0.006757333253820737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,31,0.007205333560705185
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,15,0.007018666714429855
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,31,0.006842666616042455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,63,0.006831999868154526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,63,0.007301333049933116
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,127,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,255,0.00730666642387708
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,255,0.007040000210205714
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,511,0.008746666833758354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,511,0.008682666967312494
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,1023,0.010677333921194077
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,1023,0.008922666932145754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,2047,0.012719999998807907
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,2047,0.010970667004585266
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,4095,0.027290667096773785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,4095,0.015082667271296183
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,8191,0.045738667249679565
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,8191,0.028197333216667175
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,1,0.00847999999920527
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,1,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,3,0.007914666707317034
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,3,0.00679466687142849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,7,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,7,0.00749333327015241
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,15,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,15,0.008458666503429413
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,float16,127,0.008576000109314919
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,31,0.007135999699433644
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,31,0.0075040000180403394
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,63,0.008570666735370954
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,63,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,127,0.008629333227872849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,127,0.0069919998447100324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,255,0.009125333279371262
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,15,0.13804800311724344
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,511,0.008842666943868002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,511,0.008986666798591614
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,1023,0.01091733326514562
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,1023,0.009045333291093508
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,float16,31,0.16847999890645346
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,2047,0.013178666432698568
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,4095,0.044031997521718345
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,2047,0.025637333591779072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,1,0.0075519997626543045
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,31,0.13793599605560303
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,1,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,3,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,8191,0.044938668608665466
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,3,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,7,0.006954666847983996
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,15,0.007525333513816197
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,15,0.006949333474040031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,7,0.008890666688481966
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,31,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,63,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,31,0.007040000210205714
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,127,0.007045333584149678
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,63,0.007109333450595538
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,127,0.007055999711155891
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,255,0.007237333183487256
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,255,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,511,0.00891733355820179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,511,0.008581333483258883
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,255,0.006773333375652631
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,1023,0.008639999975760778
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,2047,0.012858666479587555
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,2047,0.010773333410422007
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,4095,0.027221334477265675
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,4095,0.014677333335081736
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,1,0.008687999720374743
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,8191,0.045781334241231285
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,8191,0.0276853342851003
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,1,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,3,0.00892800030608972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,3,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,7,0.008661333471536636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,float16,8191,0.08044266700744629
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,15,0.008463999877373377
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,15,0.007007999966541926
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,31,0.008954666554927826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,31,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,63,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,127,0.008693333094318708
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,255,0.00867733359336853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,255,0.006981333096822103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,511,0.00892800030608972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,float16,15,0.16785067319869995
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,511,0.008672000219424566
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,1023,0.010826667149861654
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,float16,1023,0.010735999792814255
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,1023,0.00901333304742972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,2047,0.012784000486135483
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,2047,0.026709333062171936
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,4095,0.04404800136884054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,4095,0.027189334233601887
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,8191,0.08158933122952779
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,1,0.009098666409651438
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,4095,0.027061333258946735
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,7,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,3,0.00897066667675972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,3,0.008767999708652496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,7,0.00890666681031386
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,7,0.008725333337982496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,15,0.00891733355820179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,15,0.008853333070874214
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,31,0.008623999853928884
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,31,0.008922666932145754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,127,0.0069919998447100324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,63,0.008725333337982496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,63,0.008986666798591614
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,127,0.00874133345981439
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,127,0.008853333070874214
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,255,0.00890666681031386
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,255,0.00874133345981439
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,511,0.01231466606259346
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,511,0.010640000303586325
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,1023,0.026202666262785595
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,1023,0.012746666868527731
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,2047,0.04446933170159658
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,2047,0.02734400083621343
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,4095,0.08057599763075511
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,4095,0.04523199796676636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,1,0.006821333120266597
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,1,0.006602666651209195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,3,0.006714666883150737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,8191,0.08258666594823201
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,3,0.00679466687142849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,7,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,15,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,7,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,15,0.00679466687142849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,31,0.007050666958093643
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,31,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,63,0.006805333619316419
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,63,0.006655999769767125
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,127,0.006613333399097125
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,float16,63,0.008639999975760778
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,255,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,511,0.008485333373149237
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,1023,0.008832000195980072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,1023,0.008816000074148178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,2047,0.008912000184257826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,2047,0.008538666491707167
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,4095,0.008805333326260248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,4095,0.008581333483258883
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,8191,0.010847999403874079
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,8191,0.008810666700204214
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,16383,0.012970666090647379
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,8191,0.044768000642458596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,16383,0.012885333349307379
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,float16,8191,0.1544266641139984
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,1,0.008709333216150602
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,32767,0.019093333433071773
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,1,0.006735999758044879
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,1,0.006704000135262807
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,3,0.00702400008837382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,3,0.006655999769767125
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,7,0.006800000245372455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,127,0.006789333497484525
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,15,0.0069866664707660675
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,15,0.007045333584149678
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,31,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,31,0.006597333277265231
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,511,0.008656000097592672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,63,0.006650666395823161
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,63,0.007050666958093643
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,127,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,255,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,511,0.008767999708652496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,511,0.007002666592597961
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,1023,0.00867733359336853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,1023,0.008853333070874214
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,2047,0.00879466657837232
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,2047,0.0086666668454806
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,4095,0.008976000050703684
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,4095,0.008693333094318708
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,8191,0.010672000547250112
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,8191,0.008976000050703684
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,16383,0.014453332871198654
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,16383,0.01292266696691513
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,32767,0.017210666090250015
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,32767,0.015200000256299973
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,1,0.006730666384100914
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,1,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,3,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,3,0.006560000280539195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,7,0.007093333328763644
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,7,0.006842666616042455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,15,0.006560000280539195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,32767,0.017423999806245167
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,15,0.006810666372378667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,31,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,31,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,63,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,float16,7,0.006698666761318843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,127,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,255,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,255,0.006538666784763336
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,511,0.008858666444818178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,511,0.0068800002336502075
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,127,0.007743999982873599
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,1023,0.008522666369875273
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,1023,0.008938666433095932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,2047,0.00890666681031386
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,2047,0.008522666369875273
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,4095,0.008938666433095932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,4095,0.008778666456540426
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,8191,0.010794666906197866
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,16383,0.01303999995191892
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,32767,0.030106666187445324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,32767,0.015791999797026317
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,1,0.006981333096822103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,1,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,3,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,3,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,7,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,7,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,15,0.0069973332186539965
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,63,0.00679466687142849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,15,0.007007999966541926
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,31,0.0069440001000960665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,31,0.0069759997228781385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,float16,255,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,63,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,63,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,127,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,127,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,255,0.006762666627764702
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,255,0.006800000245372455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,511,0.009088000282645226
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,511,0.008320000022649765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,1023,0.00901333304742972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,2047,0.009072000160813332
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,2047,0.008799999952316284
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,4095,0.009072000160813332
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,16383,0.015040000279744467
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,4095,0.008762666955590248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,8191,0.01089599976936976
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,8191,0.010090666512648264
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,16383,0.014922666052977243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,16383,0.014959999670584997
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,float16,32767,0.019029332945744198
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,1,0.006949333474040031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,1,0.006805333619316419
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,3,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,3,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,7,0.007034666836261749
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,15,0.007114666824539502
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,15,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,127,0.006591999903321266
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,31,0.006746666505932808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,31,0.006602666651209195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,63,0.006730666384100914
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,63,0.006709333509206772
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,127,0.006751999879876773
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,255,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,255,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,1023,0.008858666444818178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,511,0.008586666857202848
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,511,0.0069866664707660675
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,1023,0.008885333314538002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,float16,8191,0.012725333372751871
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,1023,0.008896000062425932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,2047,0.008687999720374743
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,4095,0.00898133342464765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,4095,0.008997333546479544
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,8191,0.0129120002190272
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,16383,0.015024000157912573
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,32767,0.016864000509182613
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,16383,0.013056000073750814
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,32767,0.029674666623274486
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,32767,0.015872000406185787
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,7,0.006602666651209195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,1,0.006490666419267654
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,3,0.00707733320693175
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,3,0.007040000210205714
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,7,0.0069973332186539965
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,7,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,15,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,15,0.006730666384100914
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,31,0.007055999711155891
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,31,0.006586666529377301
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,63,0.006800000245372455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,63,0.006831999868154526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,float16,127,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,127,0.006698666761318843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,255,0.007167999943097432
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,255,0.006784000123540561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,255,0.00679466687142849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,511,0.008810666700204214
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,511,0.006954666847983996
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,1023,0.00855466661353906
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,1023,0.008618666479984919
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,2047,0.008874666566650072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,2047,0.008938666433095932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,4095,0.010735999792814255
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,4095,0.0106133334338665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,2047,0.00891733355820179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,8191,0.01246400053302447
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,8191,0.014954666296641031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,8191,0.010826667149861654
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,16383,0.028736000259717304
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,32767,0.047024001677831016
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,32767,0.029178666571776073
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,1,0.008645333349704742
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,1,0.008538666491707167
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,3,0.00860799973209699
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,7,0.008879999940594038
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,3,0.0074986666440963745
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,7,0.0069973332186539965
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,15,0.008570666735370954
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,15,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,31,0.00877333308259646
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,31,0.007530666887760162
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,63,0.008698666468262672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,63,0.008629333227872849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,127,0.008767999708652496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,127,0.007541333635648091
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,255,0.008592000231146812
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,255,0.00850133349498113
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,127,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,511,0.008901333436369896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,1023,0.011077333241701126
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,1023,0.010693332801262537
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,2047,0.013023999830087027
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,2047,0.02737066646416982
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,16383,0.015072000523408255
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,4095,0.027242665489514668
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,3,0.00901333304742972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,1,0.008853333070874214
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,1,0.008837333569924036
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,float16,1,0.006842666616042455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,7,0.008879999940594038
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,15,0.008826666822036108
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,15,0.008586666857202848
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,31,0.008976000050703684
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,31,0.008693333094318708
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,63,0.008885333314538002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,63,0.008597333605090777
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,511,0.008933333059151968
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,127,0.008805333326260248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,255,0.008922666932145754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,255,0.008623999853928884
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,511,0.010933333386977514
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,511,0.01073066641887029
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,2047,0.045066664616266884
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,1023,0.027066667874654133
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,float16,4095,0.04515733321507772
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,2047,0.02698666602373123
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,4095,0.08039466540018718
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,1,0.011551999797423681
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,4095,0.045935998360315956
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,1,0.01062400018175443
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,7,0.00902399979531765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,3,0.012874666601419449
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,3,0.010650667051474253
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,7,0.010746666540702185
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,7,0.011418666690587997
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,15,0.012639999389648438
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,15,0.010778666784365972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,31,0.010741333166758219
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,31,0.01211200033624967
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,float16,127,0.009125333279371262
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,63,0.011941333611806234
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,63,0.010981333752473196
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,127,0.010581333190202713
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,127,0.012885333349307379
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,255,0.012080000092585882
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,255,0.01089599976936976
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,511,0.027045334378878277
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,511,0.013050666699806849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,1023,0.027189334233601887
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,1023,0.012970666090647379
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,1,0.006704000135262807
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,2047,0.08218666911125183
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,3,0.006821333120266597
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,3,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,1,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,7,0.007029333462317784
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,3,0.008549333239595095
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,7,0.007045333584149678
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,15,0.00702400008837382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,15,0.007029333462317784
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,31,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,4095,0.08256533245245616
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,4095,0.15647466977437338
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,63,0.00702400008837382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,63,0.00702400008837382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,127,0.006805333619316419
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,255,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,127,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,255,0.00679466687142849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,511,0.008832000195980072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,511,0.00873066671192646
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,1023,0.008922666932145754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,2047,0.008714666590094566
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,4095,0.008922666932145754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,4095,0.009077333534757296
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,8191,0.012853333105643591
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,16383,0.015178666760524115
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,16383,0.014794666320085526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,32767,0.0312266672650973
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,32767,0.01783466711640358
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,1,0.006784000123540561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,1,0.00666133314371109
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,3,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,3,0.006565333033601443
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,7,0.00707733320693175
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,float16,1023,0.04508799811204275
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,15,0.007055999711155891
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,2047,0.04597333570321401
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,31,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,31,0.006671999891599019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,63,0.0069866664707660675
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,31,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,63,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,127,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,127,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,255,0.0068800002336502075
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,float16,1023,0.009077333534757296
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,2047,0.008885333314538002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,511,0.009589333087205887
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,8191,0.012416000167528788
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,1023,0.008837333569924036
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,1023,0.00855466661353906
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,2047,0.008938666433095932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,2047,0.008687999720374743
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,4095,0.010960000256697336
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,4095,0.010778666784365972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,8191,0.01258133351802826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,16383,0.029135999580224354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,32767,0.046853333711624146
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,7,0.006671999891599019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,32767,0.029071999092896778
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,15,0.006725333631038666
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,1,0.006666666517655055
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,1,0.0069759997228781385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,3,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,3,0.006784000123540561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,7,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,15,0.006768000001708667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,15,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,31,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,31,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,63,0.006613333399097125
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,63,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,127,0.006735999758044879
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,127,0.006762666627764702
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,255,0.00679466687142849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,255,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,511,0.008682666967312494
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,511,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,float16,8191,0.014938666174809137
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,1023,0.008837333569924036
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,16383,0.015109332899252573
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,2047,0.010778666784365972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,2047,0.00879466657837232
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,4095,0.010640000303586325
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,4095,0.01267733300725619
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,8191,0.026373334228992462
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,8191,0.01302933320403099
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,16383,0.027087998886903126
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,16383,0.045253331462542214
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,32767,0.04595733185609182
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,1,0.010858666151762009
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,7,0.006720000257094701
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,1,0.009152000149091085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,3,0.010768000036478043
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,255,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,7,0.010703999549150467
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,15,0.010901333143313726
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,511,0.006954666847983996
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,15,0.009109333157539368
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,31,0.008778666456540426
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,63,0.010608000059922537
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,1023,0.008922666932145754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,63,0.008992000172535578
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,127,0.009573333586255709
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,255,0.010693332801262537
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,255,0.009050666665037474
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,float16,32767,0.08035199840863545
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,511,0.012869333227475485
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,511,0.011125333607196808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,1023,0.027285332481066387
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,3,0.009136000027259191
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,7,0.00895999992887179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,2047,0.04593066871166229
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,2047,0.02887466549873352
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,1,0.010773333410422007
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,31,0.010725333044926325
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,1,0.012805332740147909
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,7,0.010832000523805618
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,15,0.011034666250149408
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,3,0.010677333921194077
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,7,0.012063999970753988
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,float16,127,0.010837333897749582
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,31,0.012960000584522883
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,31,0.011045332998037338
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,63,0.012714666624863943
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,63,0.010666667173306147
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,127,0.012928000340859095
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,127,0.010847999403874079
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,255,0.010751999914646149
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,1023,0.014869333555301031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,511,0.014256000518798828
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,3,0.012453333785136541
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,1023,0.02755733331044515
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,2047,0.04667200148105621
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,15,0.011946666985750198
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,1,0.01704000060757001
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,1,0.014874666929244995
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,3,0.016970666746298473
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,3,0.014938666174809137
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,7,0.016778666526079178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,7,0.01469333345691363
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,15,0.016917333006858826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,15,0.014783999572197596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,255,0.012330666184425354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,511,0.026975999275843304
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,31,0.016842667013406754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,31,0.014805333067973455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,63,0.016943999876578648
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,1023,0.04584533472855886
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,127,0.016965333372354507
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,float16,2047,0.08265600105126698
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,255,0.026357332865397137
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,255,0.014906667172908783
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,511,0.0440586656332016
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,1023,0.08132799963156383
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,1023,0.04585599899291992
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,float16,2047,0.1525973379611969
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,2047,0.08247466882069905
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,1,0.014794666320085526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,1,0.01292266696691513
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,3,0.014794666320085526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,63,0.015002666662136713
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,3,0.012714666624863943
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,7,0.014741333822409311
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,7,0.012693333129088083
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,127,0.014853333433469137
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,15,0.014885333677132925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,15,0.012741333494583765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,31,0.012831999609867731
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,63,0.014720000326633453
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,511,0.027087998886903126
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,63,0.012896000097195307
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,127,0.014831999937693277
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,127,0.012831999609867731
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,255,0.01479999969402949
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,255,0.012858666479587555
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,1,0.016927999754746754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,511,0.02775466690460841
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,1,0.015040000279744467
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,511,0.01682666689157486
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,3,0.016879999389251072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,3,0.014890667051076889
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,7,0.016965333372354507
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,7,0.014842666685581207
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,15,0.014890667051076889
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,15,0.016778666526079178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,31,0.017242666333913803
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,31,0.014874666929244995
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,63,0.016751999656359356
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,float16,31,0.014842666685581207
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,63,0.014885333677132925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,127,0.016879999389251072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,127,0.014730667074521383
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,255,0.027269333600997925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,255,0.014858666807413101
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,1,0.02736533433198929
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,511,0.027690666417280834
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,float16,511,0.04548799991607666
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,1,0.02298133323589961
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,3,0.022970666488011677
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,3,0.02720533311367035
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,7,0.02743999908367793
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,7,0.0230880007147789
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,15,0.02738133321205775
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,31,0.02309333284695943
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,63,0.02733866622050603
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,63,0.023978665471076965
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,15,0.023039999107519787
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,127,0.02934933453798294
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,31,0.027280000348885853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,127,0.023071999351183575
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,255,0.028069332242012024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,255,0.04568000137805939
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,1,0.021210665504137676
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,511,0.045834665497144066
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,float16,511,0.08205866813659668
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,1,0.01710933322707812
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,3,0.021290667355060577
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,3,0.016997333616018295
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,15,0.021327999730904896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,7,0.021615999440352123
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,7,0.01710933322707812
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,31,0.018112000077962875
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,63,0.021727999051411945
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,63,0.01693333312869072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,31,0.021989333132902782
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,127,0.021157334248224895
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,127,0.016997333616018295
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,1,0.02720000098148982
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,1,0.023370665808518726
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,255,0.017130666722853977
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,float16,255,0.028586665789286297
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,3,0.02712533374627431
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,3,0.023258666197458904
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,7,0.027503999571005504
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,7,0.023189333577950794
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,15,0.027386667827765148
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,15,0.02313599983851115
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,15,0.017279999951521557
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,31,0.027434666951497395
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,31,0.022895999252796173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,63,0.027024000883102417
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,63,0.023056000471115112
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,127,0.029482667644818623
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,127,0.023242667317390442
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,float16,255,0.046426668763160706
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,1,0.04929600159327189
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,255,0.02942399928967158
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,1,0.03976533313592275
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,3,0.048138668139775596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,3,0.039674667020638786
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,7,0.04821866750717163
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,7,0.03952533255020777
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,15,0.04851733148097992
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,31,0.04887466629346212
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,31,0.04022400081157684
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,1,0.006831999868154526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,63,0.03938666731119156
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,63,0.04933333396911621
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,1,0.006666666517655055
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,7,0.007087999954819679
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,7,0.007040000210205714
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,15,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,3,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,15,0.0069919998447100324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,31,0.00701333334048589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,127,0.03957333415746689
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,127,0.05052266518274943
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,63,0.007034666836261749
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,127,0.00679466687142849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,255,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,63,0.00707733320693175
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,127,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,255,0.00660800002515316
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,511,0.00884799969693025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,511,0.008581333483258883
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,1023,0.00878399983048439
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,float16,255,0.08611200253168742
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,1023,0.008933333059151968
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,2047,0.008858666444818178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,2047,0.008858666444818178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,4095,0.010954666882753372
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,8191,0.014789332946141561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,8191,0.01310933381319046
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,15,0.0397173340121905
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,16383,0.029088000456492107
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,16383,0.01718933383623759
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,32767,0.048351998130480446
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,32767,0.03001066545645396
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,1,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,1,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,4095,0.012805332740147909
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,3,0.006677333265542984
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,7,0.006671999891599019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,7,0.007007999966541926
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,15,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,31,0.006591999903321266
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,31,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,float16,3,0.007002666592597961
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,63,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,63,0.006735999758044879
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,127,0.006784000123540561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,127,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,255,0.0069440001000960665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,255,0.006981333096822103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,31,0.0069759997228781385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,511,0.009002666920423508
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,511,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,1023,0.009002666920423508
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,1023,0.008496000121037165
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,2047,0.010591999938090643
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,2047,0.00901333304742972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,4095,0.012800000607967377
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,8191,0.02700799951950709
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,16383,0.04430399835109711
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,16383,0.027087998886903126
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,32767,0.08096000055472057
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,32767,0.045791998505592346
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,1,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,1,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,3,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,3,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,7,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,7,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,15,0.0069440001000960665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,15,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,31,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,31,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,float16,3,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,63,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,127,0.007029333462317784
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,15,0.00679466687142849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,127,0.006784000123540561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,255,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,255,0.006666666517655055
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,511,0.0075040000180403394
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,1023,0.009674666449427605
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,1023,0.0086666668454806
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,2047,0.010981333752473196
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,2047,0.012495999534924826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,4095,0.026672000686327618
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,4095,0.010735999792814255
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,8191,0.01302933320403099
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,4095,0.013002666334311167
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,8191,0.02701866626739502
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,8191,0.044064000248909
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,255,0.04957866668701172
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,16383,0.08042666812737782
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,16383,0.04544533292452494
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,32767,0.08031466603279114
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,32767,0.15235199530919394
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,63,0.006773333375652631
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,float16,511,0.00855466661353906
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,1,0.02733866622050603
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,3,0.037530665596326195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,3,0.027456000447273254
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,1,0.037578667203585304
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,7,0.037589333951473236
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,7,0.027445333699385326
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,15,0.03687999894221624
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,15,0.02739199995994568
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,31,0.037445334096749626
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,63,0.02717866748571396
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,63,0.037317333122094475
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,31,0.027488000690937042
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,float16,127,0.037952000896135964
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,127,0.027530667682488758
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,1,0.04769066472848257
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,1,0.03941866755485535
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,3,0.04789333542188009
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,3,0.039674667020638786
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,7,0.04832000037034353
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,15,0.049925332268079124
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,7,0.03969600051641464
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,15,0.03975466638803482
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,31,0.04751466711362203
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,31,0.03949866692225138
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,63,0.03965866565704346
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,63,0.049728001157442726
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,float16,127,0.05189866820971171
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,1,0.07242133220036824
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,127,0.041589332123597465
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,1,0.08854933579762776
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,3,0.0895253320535024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,3,0.072543998559316
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,7,0.0886240005493164
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,7,0.07249600191911061
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,15,0.07212266822655995
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,31,0.07233599821726482
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,31,0.088837335507075
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,63,0.08896533648173015
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,63,0.07179200152556102
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,15,0.0886240005493164
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,float16,127,0.09237866600354512
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,127,0.07277866701285045
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,float16,1,0.0697386662165324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,1,0.04775466521581014
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,float16,3,0.06968000034491222
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,3,0.04747733473777771
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,float16,7,0.06834666430950165
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,7,0.04778666794300079
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,float16,15,0.06912533442179362
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,15,0.04795200129350027
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,float16,31,0.06824533144632976
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,63,0.04775466521581014
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,float16,63,0.06811733543872833
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,31,0.047637333472569786
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,float16,1,0.08875733613967896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,1,0.07233066856861115
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,float16,3,0.08872000376383464
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,3,0.0743999977906545
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,float16,7,0.0897813340028127
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,7,0.07418666779994965
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,float16,15,0.08950400352478027
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,15,0.0728000005086263
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,float16,31,0.08843732873598735
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,31,0.07238933444023132
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,float16,63,0.0885599950949351
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,float16,1,0.16913066307703653
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,63,0.07314666608969371
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,1,0.1358453333377838
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,float16,3,0.1684266726175944
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,3,0.13615999619166055
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,float16,7,0.1681493322054545
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,1,0.006645333642760913
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,1,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,7,0.13565333684285483
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,3,0.0069759997228781385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,7,0.006981333096822103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,3,0.006949333474040031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,7,0.006751999879876773
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,15,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,float16,15,0.16754666964213052
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,15,0.006768000001708667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,31,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,31,0.006602666651209195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,63,0.006746666505932808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,127,0.007071999832987785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,63,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,127,0.006666666517655055
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,255,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,15,0.1357919971148173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,255,0.00666133314371109
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,511,0.008752000207702318
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,511,0.008650666723648706
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,1023,0.008725333337982496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,1023,0.008912000184257826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,2047,0.010714666297038397
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,2047,0.00897066667675972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,4095,0.012730666746695837
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,4095,0.01099733387430509
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,8191,0.027482666075229645
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,float16,16383,0.04549333453178406
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,16383,0.02867199977238973
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,1,0.006682666639486949
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,float16,31,0.16806399822235107
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,3,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,31,0.13578133781750998
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,3,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,15,0.0069759997228781385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,15,0.006735999758044879
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,7,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,31,0.00702400008837382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,31,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,63,0.007045333584149678
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,63,0.006746666505932808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,127,0.007018666714429855
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,float16,63,0.16834133863449097
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,127,0.006570666407545407
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,255,0.007071999832987785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,255,0.007002666592597961
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,511,0.008805333326260248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,1023,0.008746666833758354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,2047,0.010949333508809408
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,2047,0.010821333775917688
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,1,0.006768000001708667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,4095,0.02699733277161916
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,1023,0.00922133338948091
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,4095,0.013130666067202887
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,8191,0.045514668027559914
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,8191,0.027082666754722595
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,16383,0.0461760014295578
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,16383,0.08066133161385854
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,63,0.13607466220855713
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,1,0.00867733359336853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,3,0.007034666836261749
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,3,0.008757333581646284
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,7,0.0069866664707660675
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,7,0.00679466687142849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,15,0.007002666592597961
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,15,0.008442666381597519
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,31,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,31,0.00877333308259646
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,63,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,63,0.008618666479984919
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,127,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,127,0.008789333204428354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,255,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,255,0.00874133345981439
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,7,0.0086666668454806
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,511,0.008661333471536636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,511,0.009866666669646898
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,1023,0.008965333302815756
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,1023,0.010863999525705973
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,2047,0.02601066728432973
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,2047,0.012826666235923767
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,4095,0.025749333202838898
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,4095,0.04389866689840952
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,8191,0.04409599800904592
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,8191,0.08017600079377492
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,1,0.008762666955590248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,float16,16383,0.15222932895024618
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,16383,0.08035733302434285
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,1,0.007157333195209503
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,3,0.007840000092983246
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,3,0.0069973332186539965
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,7,0.008645333349704742
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,15,0.007167999943097432
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,7,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,1,0.006688000013430913
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,31,0.008810666700204214
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,63,0.007173333317041397
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,127,0.008538666491707167
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,127,0.00702400008837382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,255,0.007098666702707608
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,255,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,511,0.008778666456540426
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,511,0.008890666688481966
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,1023,0.010661333799362183
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,2047,0.01301866645614306
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,2047,0.01091733326514562
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,4095,0.0273333340883255
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,float16,511,0.008725333337982496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,4095,0.014730667074521383
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,float16,8191,0.045968001087506614
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,8191,0.02847466617822647
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,1,0.008629333227872849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,1,0.007082666580875714
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,3,0.008581333483258883
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,3,0.0068800002336502075
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,7,0.008863999818762144
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,7,0.00655466690659523
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,15,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,15,0.008842666943868002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,15,0.006671999891599019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,31,0.007055999711155891
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,31,0.007231999809543292
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,63,0.008623999853928884
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,63,0.006773333375652631
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,127,0.008933333059151968
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,127,0.0069440001000960665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,255,0.008618666479984919
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,255,0.007285333548982938
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,1023,0.008816000074148178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,511,0.008901333436369896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,511,0.00871999996403853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,1023,0.010901333143313726
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,1023,0.009786666681369146
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,2047,0.025920001169045765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,2047,0.012608000387748083
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,4095,0.04459733267625173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,8191,0.0448586642742157
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,8191,0.08049599826335907
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,4095,0.02784000088771184
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,1,0.008901333436369896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,1,0.008901333436369896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,3,0.00873066671192646
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,7,0.008576000109314919
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,7,0.009082666908701261
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,15,0.00898133342464765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,15,0.008885333314538002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,31,0.008693333094318708
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,31,0.008943999807039896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,63,0.007040000210205714
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,63,0.008901333436369896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,63,0.008613333106040955
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,127,0.008613333106040955
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,127,0.009189333145817121
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,255,0.008890666688481966
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,511,0.01138666644692421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,511,0.010575999816258749
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,1023,0.026191999514897663
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,1023,0.013088000317414602
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,2047,0.04381866753101349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,2047,0.027029333015282948
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,4095,0.08042666812737782
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,4095,0.045509333411852516
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,8191,0.014730667074521383
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,1,0.007029333462317784
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,1,0.006602666651209195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,3,0.006954666847983996
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,8191,0.08267199993133545
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,7,0.007007999966541926
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,float16,8191,0.15387733777364096
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,7,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,15,0.00679466687142849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,31,0.0068800002336502075
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,31,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,63,0.006821333120266597
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,63,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,127,0.007146666447321574
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,3,0.0085333331177632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,127,0.006954666847983996
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,255,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,511,0.008762666955590248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,511,0.006757333253820737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,1023,0.00884799969693025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,1023,0.008816000074148178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,2047,0.00895999992887179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,2047,0.008597333605090777
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,4095,0.009008000294367472
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,4095,0.008581333483258883
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,8191,0.010879999647537867
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,8191,0.00897066667675972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,16383,0.013141332815090815
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,16383,0.012960000584522883
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,float16,32767,0.018933333456516266
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,32767,0.01714133347074191
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,1,0.006784000123540561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,255,0.0086666668454806
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,1,0.006645333642760913
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,3,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,7,0.006805333619316419
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,7,0.006570666407545407
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,15,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,15,0.006831999868154526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,31,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,31,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,63,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,63,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,127,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,127,0.006671999891599019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,255,0.006768000001708667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,255,0.006602666651209195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,511,0.009103999783595404
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,3,0.007045333584149678
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,1023,0.008703999842206636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,1023,0.007983999947706858
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,15,0.006704000135262807
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,2047,0.008826666822036108
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,4095,0.009173333023985228
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,4095,0.009312000125646591
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,8191,0.010015999898314476
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,8191,0.00897066667675972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,16383,0.01403733342885971
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,16383,0.012944000462690989
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,float16,32767,0.017258666455745697
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,float16,31,0.008629333227872849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,1,0.006949333474040031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,1,0.006618666773041089
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,3,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,3,0.006751999879876773
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,3,0.006506666541099548
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,7,0.007141333073377609
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,7,0.006671999891599019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,15,0.006640000268816948
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,15,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,31,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,31,0.006597333277265231
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,63,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,63,0.006800000245372455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,127,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,127,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,255,0.006778666749596596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,511,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,255,0.006581333155433337
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,511,0.008965333302815756
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,511,0.007002666592597961
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,2047,0.008736000085870424
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,1023,0.0086666668454806
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,1023,0.008661333471536636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,2047,0.00867733359336853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,2047,0.008992000172535578
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,4095,0.008693333094318708
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,4095,0.008736000085870424
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,8191,0.012026666353146235
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,255,0.0069973332186539965
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,8191,0.011002667248249054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,16383,0.01594666639963786
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,1,0.006693333387374878
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,32767,0.015365333606799444
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,float16,32767,0.030266667405764263
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,3,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,3,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,32767,0.015237333873907724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,7,0.00679466687142849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,15,0.007130666946371396
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,7,0.006720000257094701
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,31,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,15,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,63,0.006842666616042455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,31,0.007082666580875714
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,63,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,127,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,127,0.006842666616042455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,255,0.007055999711155891
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,255,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,511,0.00789866658548514
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,1023,0.008634666601816813
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,1023,0.008709333216150602
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,2047,0.00901333304742972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,2047,0.008565333361426989
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,4095,0.008933333059151968
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,8191,0.010751999914646149
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,8191,0.010037333394090334
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,16383,0.014773332824309668
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,32767,0.019173332800467808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,32767,0.01716800034046173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,1,0.006821333120266597
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,1,0.006677333265542984
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,3,0.00702400008837382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,3,0.006629333520929019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,7,0.006730666384100914
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,15,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,15,0.006762666627764702
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,31,0.0069759997228781385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,31,0.0069919998447100324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,63,0.006560000280539195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,63,0.006730666384100914
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,127,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,127,0.006688000013430913
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,255,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,1,0.007733333234985669
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,511,0.008522666369875273
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,511,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,1023,0.008949333180983862
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,1023,0.008837333569924036
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,2047,0.008634666601816813
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,511,0.008725333337982496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,2047,0.008762666955590248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,float16,4095,0.008885333314538002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,4095,0.008746666833758354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,8191,0.01184533288081487
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,16383,0.013728000223636627
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,8191,0.010869332899649939
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,16383,0.015200000256299973
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,16383,0.013061333447694778
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,32767,0.015210667004187902
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,1,0.006693333387374878
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,1,0.007071999832987785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,3,0.00702400008837382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,3,0.006666666517655055
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,7,0.007002666592597961
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,7,0.006762666627764702
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,15,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,15,0.007018666714429855
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,31,0.007045333584149678
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,31,0.006591999903321266
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,16383,0.012896000097195307
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,63,0.006688000013430913
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,63,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,127,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,127,0.006725333631038666
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,255,0.006821333120266597
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,255,0.006634666894872983
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,511,0.008816000074148178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,511,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,1023,0.008933333059151968
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,1023,0.00871999996403853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,4095,0.00895999992887179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,2047,0.008997333546479544
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,2047,0.008512000242869059
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,4095,0.011370666325092316
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,4095,0.010629333555698395
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,32767,0.030069333811601002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,8191,0.01413333291808764
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,float16,7,0.006730666384100914
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,16383,0.029167999823888142
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,32767,0.02942933390537898
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,float16,32767,0.046256000796953835
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,1,0.008682666967312494
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,1,0.00720000018676122
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,3,0.009039999917149544
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,3,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,7,0.008581333483258883
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,7,0.007381333038210869
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,15,0.008623999853928884
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,31,0.008709333216150602
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,31,0.007594666754206021
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,255,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,63,0.008656000097592672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,63,0.0086666668454806
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,127,0.008645333349704742
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,127,0.008672000219424566
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,255,0.008746666833758354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,255,0.008576000109314919
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,511,0.01003200002014637
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,511,0.009050666665037474
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,1023,0.012357333054145178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,2047,0.02701866626739502
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,2047,0.013381333400805792
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,4095,0.04554133117198944
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,8191,0.012618667135636011
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,16383,0.015061333775520325
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,1,0.009093333035707474
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,3,0.009093333035707474
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,1,0.00892800030608972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,7,0.00890666681031386
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,15,0.008661333471536636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,31,0.00897066667675972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,float16,15,0.008986666798591614
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,31,0.008629333227872849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,63,0.009066666786869368
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,63,0.008954666554927826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,127,0.009098666409651438
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,127,0.008559999987483025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,255,0.008645333349704742
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,255,0.008976000050703684
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,1023,0.010863999525705973
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,511,0.01108266661564509
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,511,0.01073066641887029
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,1023,0.02722666660944621
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,4095,0.02770666778087616
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,1023,0.012975999464591345
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,2047,0.045093332727750145
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,3,0.00890666681031386
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,4095,0.045781334241231285
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,7,0.008682666967312494
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,1,0.010608000059922537
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,4095,0.08159466584523518
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,1,0.010863999525705973
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,float16,15,0.009093333035707474
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,3,0.012378666549921036
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,3,0.010874666273593903
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,7,0.010842667271693548
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,7,0.012309333930412928
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,15,0.010821333775917688
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,15,0.012725333372751871
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,31,0.010703999549150467
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,31,0.012906666845083237
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,63,0.012906666845083237
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,63,0.010581333190202713
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,127,0.012634667257467905
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,255,0.010794666906197866
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,255,0.013077333569526672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,511,0.027136000494162243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,1023,0.04542933404445648
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,1023,0.027317332724730175
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,511,0.013290667285521826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,2047,0.08123733103275299
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,1,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,2047,0.027082666754722595
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,1,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,3,0.0068800002336502075
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,4095,0.08211733400821686
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,7,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,7,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,float16,4095,0.15532799561818442
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,15,0.0069919998447100324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,15,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,31,0.007034666836261749
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,63,0.007034666836261749
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,127,0.0069440001000960665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,31,0.007135999699433644
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,127,0.006677333265542984
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,255,0.008037333066264788
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,255,0.00660800002515316
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,511,0.008767999708652496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,1023,0.00914666677514712
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,1023,0.008896000062425932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,2047,0.00895999992887179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,2047,0.0085333331177632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,127,0.010586666564146677
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,4095,0.00922133338948091
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,4095,0.008943999807039896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,8191,0.012757333616415659
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,8191,0.012965332716703415
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,16383,0.016250666230916977
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,16383,0.014746667196353277
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,32767,0.017898666361967724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,1,0.007007999966541926
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,3,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,2047,0.046112000942230225
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,3,0.0068800002336502075
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,7,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,3,0.007093333328763644
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,7,0.006730666384100914
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,15,0.006831999868154526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,15,0.007034666836261749
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,31,0.007120000198483467
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,63,0.006805333619316419
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,63,0.006677333265542984
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,63,0.006591999903321266
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,127,0.0069973332186539965
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,255,0.006805333619316419
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,255,0.006634666894872983
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,511,0.008623999853928884
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,511,0.007040000210205714
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,511,0.00850133349498113
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,1023,0.008912000184257826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,1023,0.008933333059151968
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,2047,0.00874133345981439
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,2047,0.008661333471536636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,4095,0.011018666128317514
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,4095,0.010677333921194077
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,8191,0.012773333738247553
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,16383,0.029343999922275543
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,16383,0.015082667271296183
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,32767,0.047093331813812256
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,32767,0.029669334491093952
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,1,0.007391999786098798
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,float16,32767,0.031189332405726116
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,3,0.007002666592597961
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,3,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,1,0.006720000257094701
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,7,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,15,0.00701333334048589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,15,0.006784000123540561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,31,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,31,0.006789333497484525
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,63,0.006800000245372455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,127,0.007029333462317784
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,31,0.00679466687142849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,127,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,255,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,255,0.006591999903321266
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,511,0.008752000207702318
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,511,0.006949333474040031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,1023,0.008645333349704742
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,2047,0.01098666712641716
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,2047,0.008586666857202848
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,8191,0.01358933374285698
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,4095,0.013034666577974955
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,4095,0.010725333044926325
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,8191,0.01314666618903478
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,16383,0.044495999813079834
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,1,0.006575999781489372
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,16383,0.027269333600997925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,32767,0.0802293320496877
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,7,0.007151999821265538
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,1,0.01098666712641716
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,1,0.00891733355820179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,63,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,3,0.010858666151762009
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,3,0.008976000050703684
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,float16,127,0.007055999711155891
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,7,0.010832000523805618
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,7,0.008986666798591614
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,15,0.01080000028014183
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,1023,0.008938666433095932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,15,0.00903466654320558
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,31,0.010911999891201654
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,31,0.008874666566650072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,63,0.010858666151762009
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,63,0.00897066667675972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,127,0.01097600037852923
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,127,0.009130666653315226
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,float16,8191,0.027098665634791057
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,255,0.010709332923094431
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,511,0.012960000584522883
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,32767,0.04550399879614512
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,511,0.011349332829316458
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,1023,0.027386667827765148
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,1023,0.01505600040157636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,float16,2047,0.04612799982229868
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,1,0.01128000020980835
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,2047,0.02828266719977061
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,1,0.011066666493813196
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,3,0.012784000486135483
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,7,0.011002667248249054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,3,0.010709332923094431
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,15,0.010890666395425797
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,7,0.012815999488035837
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,31,0.012714666624863943
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,31,0.01073066641887029
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,15,0.012928000340859095
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,63,0.012815999488035837
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,255,0.008885333314538002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,127,0.01267733300725619
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,255,0.011877333124478659
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,255,0.010703999549150467
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,511,0.027061333258946735
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,511,0.013088000317414602
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,2047,0.08226666847864787
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,float16,1023,0.04567466676235199
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,1023,0.02735466758410136
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,2047,0.04679466784000397
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,1,0.016832000265518825
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,3,0.016800000021855038
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,7,0.014912000546852747
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,15,0.017210666090250015
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,3,0.014682666709025701
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,7,0.016879999389251072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,15,0.014671999961137772
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,31,0.01692266638080279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,63,0.010992000500361124
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,31,0.014746667196353277
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,63,0.014666666587193808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,63,0.017103999853134155
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,127,0.01684800038735072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,127,0.014789332946141561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,255,0.014698666830857595
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,255,0.02700799951950709
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,1023,0.08052800099054973
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,1023,0.04608533283074697
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,511,0.02754666656255722
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,511,0.045114666223526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,1,0.015157333264748255
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,1,0.014762666076421738
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,float16,2047,0.15294933319091797
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,1,0.012736000120639801
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,7,0.014912000546852747
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,127,0.01119999960064888
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,3,0.014917333920796713
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,3,0.012608000387748083
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,7,0.012741333494583765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,15,0.014789332946141561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,15,0.012784000486135483
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,31,0.014864000181357065
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,31,0.012917333592971167
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,63,0.014725333700577417
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,63,0.012634667257467905
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,127,0.014709333578745524
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,127,0.012986666212479273
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,255,0.014773332824309668
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,1,0.01699200024207433
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,255,0.012954667210578918
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,float16,511,0.027669332921504974
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,511,0.016704000532627106
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,2047,0.08288000027338664
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,1,0.014736000448465347
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,3,0.016778666526079178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,3,0.01479999969402949
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,7,0.01684800038735072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,7,0.015029333531856537
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,31,0.014831999937693277
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,31,0.01703466723362605
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,63,0.016810666769742966
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,15,0.017237332959969837
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,15,0.015008000036080679
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,63,0.014922666052977243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,127,0.017125333348910015
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,127,0.01488000030318896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,1,0.0273333340883255
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,255,0.02720000098148982
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,1,0.022954667607943218
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,255,0.015066667149464289
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,float16,511,0.04542933404445648
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,511,0.028042666614055634
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,3,0.027317332724730175
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,3,0.023061332603295643
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,7,0.02738133321205775
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,15,0.027248000105222065
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,7,0.023247999449570973
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,31,0.02317333221435547
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,15,0.02345066765944163
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,63,0.027456000447273254
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,63,0.023045333723227184
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,31,0.027450665831565857
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,127,0.02917333443959554
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,127,0.022991999983787537
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,255,0.045461331804593406
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,1,0.022250667214393616
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,511,0.04643199841181437
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,float16,511,0.08145600060621898
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,1,0.016778666526079178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,3,0.023130667706330616
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,3,0.017162666966517765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,7,0.022175999979178112
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,7,0.016837333639462788
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,31,0.021738665799299877
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,31,0.017050666113694508
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,255,0.027930667002995808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,15,0.022917332748572033
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,63,0.02216000109910965
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,63,0.016735999534527462
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,15,0.016927999754746754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,127,0.022431999444961548
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,127,0.016832000265518825
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,float16,255,0.02739733209212621
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,1,0.027119999130566914
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,255,0.017125333348910015
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,1,0.023973333338896435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,3,0.027306665976842243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,3,0.023168000082174938
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,7,0.02719466636578242
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,7,0.023039999107519787
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,15,0.023056000471115112
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,15,0.027386667827765148
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,31,0.023215999205907185
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,63,0.027280000348885853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,31,0.02752000093460083
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,63,0.02316266546646754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,127,0.029690665503342945
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,127,0.022944000860055287
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,1,0.04926399886608124
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,1,0.03950933367013931
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,float16,255,0.04580800235271454
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,255,0.029098667204380035
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,3,0.03956266740957896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,7,0.04784533381462097
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,7,0.03932799895604452
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,15,0.04849599798520406
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,15,0.039359999199708305
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,31,0.0489333321650823
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,63,0.04972266654173533
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,31,0.03942933430274328
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,63,0.03949866692225138
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,3,0.04879466692606608
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,1,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,3,0.007061333085099856
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,7,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,1,0.007189333438873291
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,7,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,15,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,3,0.0069973332186539965
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,15,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,31,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,127,0.04038399954636892
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,63,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,127,0.051407997806866966
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,127,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,255,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,511,0.008901333436369896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,511,0.008656000097592672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,1023,0.008896000062425932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,127,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,255,0.0069973332186539965
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,2047,0.009103999783595404
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,2047,0.008639999975760778
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,float16,255,0.08265600105126698
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,4095,0.011018666128317514
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,8191,0.014783999572197596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,8191,0.012965332716703415
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,16383,0.0170666662355264
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,255,0.04791999856630961
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,32767,0.04820266862710317
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,32767,0.031167998909950256
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,1,0.007045333584149678
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,3,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,16383,0.029743999242782593
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,7,0.006842666616042455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,7,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,15,0.006746666505932808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,1,0.006730666384100914
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,15,0.006629333520929019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,31,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,31,0.006581333155433337
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,63,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,63,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,31,0.00666133314371109
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,127,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,63,0.007029333462317784
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,127,0.006458666796485583
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,255,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,511,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,1023,0.008805333326260248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,1023,0.0085333331177632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,1023,0.008709333216150602
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,2047,0.010661333799362183
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,2047,0.00898133342464765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,4095,0.012975999464591345
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,float16,4095,0.012800000607967377
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,4095,0.010784000158309937
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,8191,0.027274665733178455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,8191,0.013061333447694778
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,16383,0.027024000883102417
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,16383,0.04443199932575226
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,32767,0.0811413327852885
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,32767,0.04578666885693868
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,1,0.006751999879876773
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,3,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,3,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,3,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,7,0.006831999868154526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,15,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,15,0.006751999879876773
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,31,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,63,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,63,0.006671999891599019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,127,0.006730666384100914
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,255,0.006528000036875407
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,float16,511,0.008997333546479544
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,255,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,255,0.006613333399097125
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,511,0.007594666754206021
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,1023,0.008912000184257826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,1023,0.00892800030608972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,2047,0.011157333850860596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,4095,0.012938667088747025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,1,0.007055999711155891
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,8191,0.044112001856168113
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,7,0.006682666639486949
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,16383,0.08038933575153351
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,16383,0.04484800000985464
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,31,0.006581333155433337
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,32767,0.1527786652247111
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,127,0.007104000076651573
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,511,0.008890666688481966
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,2047,0.010533332824707031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,float16,4095,0.027045334378878277
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,8191,0.027082666754722595
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,1,0.037392000357309975
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,32767,0.08092266817887624
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,1,0.02717333287000656
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,3,0.03758399933576584
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,3,0.0271573339899381
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,7,0.03722666700681051
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,7,0.027285332481066387
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,15,0.03738666574160258
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,15,0.02699200063943863
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,31,0.037248000502586365
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,31,0.027162666122118633
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,63,0.03770133356253306
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,float16,127,0.03754133234421412
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,63,0.027162666122118633
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,127,0.027322667340437572
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,1,0.04789333542188009
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,1,0.03965333352486292
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,3,0.04934933284918467
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,3,0.03975466638803482
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,7,0.049498667319615684
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,7,0.0396373321612676
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,15,0.04849599798520406
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,15,0.039647998909155525
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,31,0.04786666731039683
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,31,0.03952533255020777
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,63,0.04862933357556661
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,63,0.039605334401130676
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,1,0.07246399919191997
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,127,0.04163199911514918
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,1,0.0886400043964386
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,float16,127,0.05157333115736643
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,3,0.0888213316599528
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,3,0.0724533349275589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,7,0.08840533097585042
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,7,0.07211733361085255
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,15,0.08872000376383464
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,15,0.07230933507283528
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,31,0.08895466725031535
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,31,0.07235733171304067
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,63,0.08869333068529765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,63,0.07203199962774913
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,float16,127,0.09257066249847412
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,127,0.07260799904664357
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,float16,1,0.06932266553243001
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,1,0.04774933556715647
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,float16,3,0.06939200063546498
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,7,0.0480373352766037
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,3,0.04766400158405304
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,float16,15,0.06993600229422252
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,15,0.047450666626294456
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,float16,7,0.06841066479682922
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,float16,31,0.06922666728496552
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,31,0.04638933142026266
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,float16,63,0.06877333422501881
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,63,0.04781333108743032
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,float16,1,0.0888213316599528
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,1,0.07249066730340321
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,float16,3,0.08867200215657552
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,3,0.07258666555086772
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,float16,7,0.0890826682249705
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,7,0.07445866862932841
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,float16,15,0.0890773336092631
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,15,0.07378666599591573
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,float16,31,0.08845333258310954
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,float16,63,0.08778132994969685
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,63,0.07255466779073079
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,float16,1,0.16832000017166138
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,1,0.13615467151006064
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,31,0.0730506678422292
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,float16,3,0.168229341506958
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,3,0.13615467151006064
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,float16,7,0.1679733395576477
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,1,0.007114666824539502
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,1,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,3,0.006831999868154526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,7,0.13734400272369385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,float16,15,0.1665066679318746
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,15,0.13580800096193948
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,7,0.007109333450595538
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,7,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,15,0.006677333265542984
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,15,0.006842666616042455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,31,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,31,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,63,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,63,0.006773333375652631
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,127,0.007711999739209811
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,127,0.0068800002336502075
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,255,0.007290666922926903
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,255,0.007125333572427432
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,511,0.008586666857202848
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,511,0.008639999975760778
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,31,0.1362399955590566
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,1023,0.010725333044926325
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,1023,0.008634666601816813
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,2047,0.01089599976936976
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,2047,0.009312000125646591
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,4095,0.012970666090647379
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,8191,0.027215999861558277
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,float16,16383,0.04574400186538696
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,4095,0.010826667149861654
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,16383,0.02867199977238973
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,1,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,1,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,3,0.0069866664707660675
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,3,0.006698666761318843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,3,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,7,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,7,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,15,0.0069440001000960665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,15,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,31,0.006709333509206772
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,31,0.006538666784763336
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,63,0.006597333277265231
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,127,0.006704000135262807
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,127,0.006591999903321266
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,255,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,255,0.006842666616042455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,511,0.008767999708652496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,511,0.007647999872763951
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,1023,0.009658666948477427
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,float16,31,0.16791999340057373
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,2047,0.01102399950226148
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,2047,0.010565333068370819
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,4095,0.027109332382678986
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,8191,0.045184001326560974
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,8191,0.02699733277161916
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,16383,0.08049599826335907
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,8191,0.014794666320085526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,16383,0.04553600152333578
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,1,0.008805333326260248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,float16,63,0.16870399316151938
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,3,0.008112000301480293
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,3,0.0068800002336502075
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,7,0.008661333471536636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,7,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,float16,63,0.006842666616042455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,15,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,31,0.008634666601816813
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,31,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,63,0.008362666393319765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,63,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,127,0.008549333239595095
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,15,0.008661333471536636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,127,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,1023,0.008698666468262672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,255,0.008645333349704742
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,255,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,511,0.008725333337982496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,511,0.008869333192706108
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,1023,0.010768000036478043
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,1023,0.008672000219424566
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,4095,0.012885333349307379
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,2047,0.012970666090647379
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,4095,0.026154667139053345
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,8191,0.04429866870244344
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,8191,0.08020799855391185
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,1,0.007018666714429855
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,16383,0.15236799915631613
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,1,0.006831999868154526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,16383,0.08042133351167043
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,1,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,3,0.006805333619316419
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,3,0.006773333375652631
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,7,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,7,0.007098666702707608
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,15,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,15,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,31,0.006949333474040031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,31,0.007178666690985362
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,63,0.13588800032933554
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,63,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,127,0.007696000238259633
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,127,0.006735999758044879
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,255,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,511,0.00898133342464765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,511,0.008703999842206636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,1023,0.010480000327030817
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,1023,0.009050666665037474
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,2047,0.010762666662534079
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,2047,0.009322666873534521
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,4095,0.012671999633312225
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,2047,0.025589334468046825
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,8191,0.027658666173617046
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,float16,4095,0.04353066782156626
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,1,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,16383,0.029365333418051403
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,16383,0.045834665497144066
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,1,0.006741333131988843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,3,0.006949333474040031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,7,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,15,0.006810666372378667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,15,0.006698666761318843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,31,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,31,0.006591999903321266
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,float16,63,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,63,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,63,0.008005333443482717
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,127,0.006949333474040031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,127,0.006624000146985054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,255,0.007007999966541926
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,255,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,511,0.008687999720374743
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,1023,0.009205333267649015
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,1023,0.008618666479984919
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,2047,0.011055999745925268
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,2047,0.010672000547250112
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,4095,0.011146667102972666
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,8191,0.014975999792416891
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,255,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,4095,0.027221334477265675
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,4095,0.012863999853531519
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,8191,0.02701866626739502
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,8191,0.04555733501911163
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,float16,16383,0.08251200119654338
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,3,0.006688000013430913
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,7,0.006640000268816948
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,1,0.008527999743819237
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,1,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,3,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,3,0.00884799969693025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,7,0.0085333331177632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,7,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,15,0.008736000085870424
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,15,0.006613333399097125
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,31,0.00892800030608972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,31,0.006981333096822103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,63,0.008527999743819237
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,63,0.006741333131988843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,127,0.008752000207702318
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,127,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,255,0.0086666668454806
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,511,0.00871999996403853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,511,0.008703999842206636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,1023,0.010890666395425797
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,1023,0.009018666421373686
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,2047,0.012879999975363413
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,4095,0.04378666480382284
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,511,0.008266666904091835
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,4095,0.02703999976317088
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,8191,0.045370668172836304
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,8191,0.08066666622956593
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,16383,0.15204800168673197
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,1,0.008912000184257826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,1,0.0086666668454806
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,16383,0.08228800197442372
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,3,0.009039999917149544
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,3,0.00873066671192646
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,7,0.008874666566650072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,15,0.008997333546479544
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,15,0.008725333337982496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,31,0.008853333070874214
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,16383,0.045850664377212524
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,31,0.008709333216150602
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,63,0.008474666625261307
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,63,0.008842666943868002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,127,0.008752000207702318
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,255,0.008976000050703684
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,255,0.008682666967312494
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,255,0.006805333619316419
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,511,0.011183999478816986
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,511,0.01073066641887029
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,float16,2047,0.025792000194390614
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,1023,0.026335999369621277
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,2047,0.043925335009892784
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,2047,0.027269333600997925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,4095,0.04524266719818115
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,4095,0.0805920014778773
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,1,0.006821333120266597
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,8191,0.1534293293952942
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,8191,0.08080533146858215
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,3,0.0069866664707660675
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,3,0.006789333497484525
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,16383,0.15838932991027832
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,float16,16383,0.2997066577275594
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,7,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,15,0.007146666447321574
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,31,0.006842666616042455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,31,0.006821333120266597
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,63,0.0069866664707660675
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,63,0.007146666447321574
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,127,0.006704000135262807
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,127,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,255,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,127,0.008687999720374743
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,255,0.00702400008837382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,511,0.008581333483258883
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,1023,0.009039999917149544
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,1023,0.008613333106040955
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,2047,0.009130666653315226
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,2047,0.008821333448092142
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,4095,0.00895999992887179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,4095,0.008869333192706108
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,1023,0.012986666212479273
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,8191,0.008757333581646284
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,16383,0.013023999830087027
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,32767,0.017173333714405697
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,32767,0.016917333006858826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,65535,0.025093334416548412
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,1,0.006597333277265231
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,65535,0.0252960001428922
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,1,0.006784000123540561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,1,0.006640000268816948
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,7,0.008634666601816813
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,7,0.006735999758044879
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,3,0.006949333474040031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,3,0.006629333520929019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,7,0.006688000013430913
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,7,0.006778666749596596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,15,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,15,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,31,0.006575999781489372
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,31,0.006677333265542984
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,63,0.00666133314371109
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,63,0.006650666395823161
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,127,0.008592000231146812
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,127,0.006565333033601443
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,255,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,511,0.008623999853928884
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,511,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,1023,0.007146666447321574
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,2047,0.009061333412925402
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,2047,0.008336000144481659
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,4095,0.008885333314538002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,511,0.008661333471536636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,4095,0.008629333227872849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,8191,0.010970667004585266
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,8191,0.008954666554927826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,16383,0.012719999998807907
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,16383,0.013066666821638743
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,32767,0.017125333348910015
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,32767,0.016783999900023144
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,65535,0.021253332495689392
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,8191,0.01099733387430509
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,65535,0.01929066702723503
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,1,0.006671999891599019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,float16,16383,0.01293333371480306
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,3,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,3,0.006565333033601443
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,7,0.006671999891599019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,7,0.006757333253820737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,15,0.006666666517655055
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,15,0.00702400008837382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,31,0.00655466690659523
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,31,0.006842666616042455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,63,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,63,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,127,0.006618666773041089
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,127,0.006746666505932808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,255,0.00701333334048589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,255,0.006645333642760913
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,511,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,1023,0.008618666479984919
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,1023,0.008912000184257826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,2047,0.008618666479984919
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,15,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,2047,0.009002666920423508
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,4095,0.008858666444818178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,4095,0.00860799973209699
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,8191,0.010175999874869982
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,255,0.006560000280539195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,16383,0.014240000396966934
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,16383,0.013194666554530462
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,32767,0.0170666662355264
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,32767,0.014938666174809137
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,65535,0.033615998923778534
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,65535,0.017077332983414333
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,1,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,1,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,3,0.006720000257094701
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,3,0.006714666883150737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,7,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,1,0.006746666505932808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,7,0.006602666651209195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,15,0.007061333085099856
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,15,0.007029333462317784
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,31,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,31,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,63,0.006778666749596596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,63,0.006773333375652631
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,127,0.006730666384100914
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,255,0.0069866664707660675
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,255,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,511,0.008703999842206636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,511,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,float16,511,0.008687999720374743
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,1023,0.00878399983048439
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,1023,0.008709333216150602
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,2047,0.00890666681031386
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,2047,0.008890666688481966
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,4095,0.00892800030608972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,4095,0.008874666566650072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,8191,0.011066666493813196
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,8191,0.00867733359336853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,16383,0.014949332922697067
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,16383,0.013125333935022354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,32767,0.03001066545645396
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,float16,1023,0.00891733355820179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,1,0.007237333183487256
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,65535,0.03143999973932902
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,65535,0.04762666424115499
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,3,0.007040000210205714
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,7,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,7,0.006714666883150737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,15,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,31,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,31,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,63,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,63,0.0069440001000960665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,127,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,127,0.00679466687142849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,255,0.006757333253820737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,255,0.006613333399097125
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,511,0.008597333605090777
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,511,0.007365333537260692
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,1023,0.008725333337982496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,127,0.006655999769767125
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,1023,0.008639999975760778
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,2047,0.008874666566650072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,2047,0.008762666955590248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,4095,0.008725333337982496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,4095,0.008863999818762144
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,8191,0.010784000158309937
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,8191,0.009002666920423508
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,16383,0.014752000570297241
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,16383,0.013050666699806849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,32767,0.018874666343132656
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,32767,0.01748266691962878
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,65535,0.022986667851607006
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,65535,0.020629333953062694
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,1,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,1,0.006650666395823161
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,3,0.006741333131988843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,float16,8191,0.013125333935022354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,3,0.006650666395823161
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,7,0.006768000001708667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,7,0.006618666773041089
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,15,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,15,0.006517333288987477
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,31,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,31,0.006565333033601443
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,63,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,63,0.006341333190600078
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,127,0.006789333497484525
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,127,0.006613333399097125
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,255,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,255,0.006618666773041089
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,float16,15,0.006746666505932808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,511,0.008789333204428354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,511,0.007029333462317784
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,1023,0.009392000113924345
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,1023,0.008650666723648706
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,2047,0.00884799969693025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,2047,0.00874133345981439
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,4095,0.00884799969693025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,4095,0.008512000242869059
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,8191,0.010677333921194077
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,8191,0.009018666421373686
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,16383,0.014858666807413101
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,16383,0.013034666577974955
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,32767,0.015072000523408255
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,65535,0.03148266673088074
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,65535,0.01720533271630605
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,1,0.0069919998447100324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,1,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,3,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,3,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,7,0.007104000076651573
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,7,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,15,0.007002666592597961
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,15,0.006800000245372455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,31,0.00666133314371109
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,31,0.006831999868154526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,63,0.006773333375652631
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,63,0.006618666773041089
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,127,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,255,0.006666666517655055
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,255,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,511,0.008650666723648706
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,511,0.0068800002336502075
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,1023,0.008656000097592672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,1023,0.008789333204428354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,2047,0.008757333581646284
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,1,0.006666666517655055
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,3,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,4095,0.008709333216150602
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,8191,0.011973333855470022
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,8191,0.01099733387430509
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,16383,0.015130666395028433
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,16383,0.013013333082199097
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,32767,0.029690665503342945
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,32767,0.015216000378131866
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,65535,0.03107733279466629
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,1,0.0069440001000960665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,3,0.006666666517655055
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,3,0.006565333033601443
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,127,0.006575999781489372
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,7,0.006810666372378667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,7,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,15,0.006789333497484525
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,15,0.007082666580875714
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,31,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,31,0.006602666651209195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,4095,0.008597333605090777
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,63,0.006517333288987477
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,63,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,127,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,127,0.006831999868154526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,255,0.0069919998447100324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,255,0.006597333277265231
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,511,0.008645333349704742
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,511,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,1023,0.00879466657837232
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,1023,0.00854399986565113
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,float16,65535,0.0481279989083608
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,2047,0.00891733355820179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,2047,0.008517333616813024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,1,0.006602666651209195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,4095,0.010725333044926325
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,4095,0.010746666540702185
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,8191,0.012624000509579977
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,8191,0.01481066644191742
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,16383,0.014778666198253632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,16383,0.029440000653266907
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,32767,0.029018667836983997
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,32767,0.04682666560014089
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,float16,65535,0.08283733328183492
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,65535,0.04752533137798309
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,1,0.008240000034372011
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,1,0.007034666836261749
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,3,0.008698666468262672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,3,0.006784000123540561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,7,0.007205333560705185
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,7,0.007007999966541926
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,15,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,15,0.0068800002336502075
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,31,0.00731733317176501
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,32767,0.015967999895413715
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,31,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,63,0.007280000175038974
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,63,0.006842666616042455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,127,0.008639999975760778
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,127,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,255,0.006773333375652631
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,255,0.008896000062425932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,511,0.008832000195980072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,1023,0.010816000401973724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,1023,0.008885333314538002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,float16,32767,0.01717866708834966
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,2047,0.012741333494583765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,2047,0.011130666981140772
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,4095,0.02720000098148982
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,8191,0.045408000548680626
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,8191,0.028421332438786823
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,1,0.008789333204428354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,1,0.007029333462317784
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,3,0.008650666723648706
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,3,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,7,0.00879466657837232
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,7,0.006789333497484525
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,15,0.008602666358153025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,15,0.007040000210205714
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,31,0.008682666967312494
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,31,0.006768000001708667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,63,0.008474666625261307
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,63,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,127,0.008762666955590248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,127,0.006602666651209195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,2047,0.008687999720374743
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,255,0.008602666358153025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,255,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,511,0.00903466654320558
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,511,0.008512000242869059
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,float16,511,0.0086666668454806
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,1023,0.008842666943868002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,2047,0.026352000733216602
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,4095,0.04436799883842468
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,4095,0.027066667874654133
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,2047,0.012698666503032049
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,8191,0.08162666857242584
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,1,0.008938666433095932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,4095,0.015029333531856537
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,3,0.00903466654320558
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,8191,0.04555733501911163
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,7,0.008863999818762144
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,1,0.00903466654320558
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,3,0.008863999818762144
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,15,0.00877333308259646
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,15,0.008885333314538002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,31,0.009103999783595404
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,31,0.008618666479984919
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,63,0.008682666967312494
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,127,0.008986666798591614
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,127,0.00891733355820179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,255,0.008997333546479544
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,255,0.00860799973209699
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,511,0.012949333836634954
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,511,0.010832000523805618
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,2047,0.04437333345413208
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,1023,0.026943999032179516
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,1023,0.013077333569526672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,2047,0.027119999130566914
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,4095,0.08088533580303192
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,4095,0.045653333266576133
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,1,0.012773333738247553
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,1,0.011018666128317514
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,3,0.012847999731699625
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,8191,0.15411200126012167
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,3,0.010703999549150467
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,7,0.009050666665037474
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,7,0.011600000162919363
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,7,0.010768000036478043
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,15,0.010714666297038397
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,15,0.013082666943470636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,31,0.012762666990359625
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,float16,63,0.008826666822036108
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,63,0.012890666723251343
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,63,0.010746666540702185
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,127,0.013061333447694778
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,127,0.010757333288590113
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,255,0.010709332923094431
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,255,0.012576000144084295
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,511,0.02720533311367035
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,1023,0.04539200166861216
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,1023,0.027258666853109997
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,float16,1023,0.011130666981140772
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,2047,0.08176533381144206
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,4095,0.15606932838757834
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,4095,0.08259200056393941
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,1,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,1,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,3,0.007029333462317784
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,3,0.0068800002336502075
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,float16,8191,0.3062719901402791
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,7,0.006831999868154526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,7,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,15,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,31,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,15,0.007034666836261749
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,31,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,63,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,31,0.01073066641887029
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,63,0.00701333334048589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,127,0.006655999769767125
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,127,0.0068800002336502075
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,255,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,255,0.006954666847983996
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,511,0.00855466661353906
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,1023,0.00898133342464765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,1023,0.008586666857202848
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,2047,0.009109333157539368
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,2047,0.008549333239595095
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,4095,0.008869333192706108
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,4095,0.008586666857202848
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,8191,0.010805333654085795
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,8191,0.00897066667675972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,511,0.014127999544143677
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,16383,0.015194666882356008
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,16383,0.014773332824309668
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,32767,0.016778666526079178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,65535,0.03275199979543686
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,float16,32767,0.018965333700180054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,65535,0.01922133316596349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,1,0.006682666639486949
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,1,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,2047,0.04558399816354116
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,3,0.006778666749596596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,7,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,15,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,7,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,15,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,31,0.006709333509206772
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,8191,0.1569653352101644
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,31,0.006575999781489372
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,63,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,63,0.006698666761318843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,127,0.006805333619316419
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,255,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,255,0.006586666529377301
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,511,0.008725333337982496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,1023,0.010399999717871347
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,1023,0.00867733359336853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,2047,0.00897066667675972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,4095,0.008799999952316284
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,4095,0.008703999842206636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,511,0.007610666876037915
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,8191,0.012869333227475485
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,8191,0.010960000256697336
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,16383,0.015013333410024643
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,16383,0.01293333371480306
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,32767,0.02972800036271413
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,32767,0.015504000087579092
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,65535,0.04823466638724009
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,65535,0.030373332401116688
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,1,0.00679466687142849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,1,0.006634666894872983
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,3,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,3,0.006517333288987477
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,7,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,7,0.006645333642760913
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,15,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,31,0.006805333619316419
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,3,0.007098666702707608
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,63,0.007135999699433644
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,63,0.006570666407545407
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,127,0.006842666616042455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,127,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,255,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,255,0.007055999711155891
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,511,0.008736000085870424
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,511,0.007071999832987785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,511,0.007104000076651573
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,1023,0.008634666601816813
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,1023,0.008762666955590248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,2047,0.008639999975760778
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,2047,0.008650666723648706
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,4095,0.01097600037852923
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,4095,0.01073066641887029
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,8191,0.01381333296497663
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,8191,0.012736000120639801
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,16383,0.029264000554879505
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,16383,0.014826666563749313
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,32767,0.04758933186531067
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,32767,0.029135999580224354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,float16,65535,0.08260799944400787
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,65535,0.04797333478927612
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,1,0.006751999879876773
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,1,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,15,0.006693333387374878
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,3,0.006693333387374878
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,3,0.007029333462317784
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,31,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,7,0.007002666592597961
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,7,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,15,0.007034666836261749
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,31,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,31,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,8191,0.08302400012811025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,63,0.007002666592597961
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,127,0.006634666894872983
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,127,0.006725333631038666
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,255,0.006624000146985054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,255,0.0068800002336502075
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,float16,127,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,511,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,511,0.00884799969693025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,1023,0.00871999996403853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,2047,0.010901333143313726
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,2047,0.008965333302815756
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,4095,0.012698666503032049
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,4095,0.010735999792814255
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,8191,0.012847999731699625
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,8191,0.027386667827765148
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,16383,0.04541866481304169
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,16383,0.02716800073782603
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,32767,0.08072533210118611
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,32767,0.0458133320013682
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,65535,0.15330132842063904
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,65535,0.08297599852085114
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,1,0.008687999720374743
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,1,0.0086666668454806
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,3,0.008938666433095932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,15,0.006655999769767125
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,63,0.006831999868154526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,7,0.00878399983048439
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,7,0.007093333328763644
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,15,0.008858666444818178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,float16,1023,0.00898133342464765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,2047,0.008698666468262672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,31,0.00884799969693025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,31,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,63,0.00860799973209699
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,63,0.007146666447321574
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,127,0.0086666668454806
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,255,0.008954666554927826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,255,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,511,0.009029333169261614
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,511,0.00898133342464765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,1023,0.01109333336353302
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,1023,0.010992000500361124
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,2047,0.027066667874654133
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,2047,0.013717333475748697
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,3,0.00877333308259646
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,float16,4095,0.04548266530036926
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,4095,0.02735999971628189
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,1,0.008879999940594038
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,3,0.00933333362142245
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,3,0.008816000074148178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,15,0.008853333070874214
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,15,0.008949333180983862
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,15,0.008682666967312494
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,7,0.009045333291093508
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,31,0.008949333180983862
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,7,0.00877333308259646
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,31,0.008703999842206636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,127,0.00731733317176501
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,63,0.008863999818762144
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,63,0.008634666601816813
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,127,0.008938666433095932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,127,0.008629333227872849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,255,0.008954666554927826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,255,0.008698666468262672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,511,0.01181866725285848
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,511,0.010858666151762009
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,1023,0.027082666754722595
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,1023,0.012986666212479273
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,4095,0.08201066652933757
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,4095,0.04552533229192098
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,1,0.008912000184257826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,1,0.01198400060335795
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,2047,0.027376001079877216
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,3,0.010725333044926325
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,7,0.012330666184425354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,1,0.010698666175206503
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,3,0.011722666521867117
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,7,0.010938666760921478
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,15,0.012213333199421564
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,31,0.010581333190202713
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,15,0.0107893335322539
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,63,0.010608000059922537
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,63,0.01267733300725619
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,127,0.010575999816258749
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,255,0.011941333611806234
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,255,0.01081066702802976
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,511,0.02700799951950709
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,511,0.013658666362365087
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,1023,0.04562133550643921
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,2047,0.08187733093897502
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,2047,0.0458133320013682
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,float16,2047,0.04576000074545542
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,4095,0.15574933091799417
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,1,0.017210666090250015
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,1,0.01479999969402949
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,4095,0.08406399687131245
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,3,0.014858666807413101
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,3,0.01682666689157486
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,7,0.016938666502634685
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,15,0.0170666662355264
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,127,0.011760000139474869
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,15,0.014965333044528961
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,31,0.016895999511082966
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,63,0.01488000030318896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,63,0.016895999511082966
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,127,0.017082666357358296
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,127,0.014991999914248785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,1023,0.02714666724205017
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,255,0.014789332946141561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,255,0.02585600068171819
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,511,0.044480000933011375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,511,0.027098665634791057
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,1023,0.08083199958006541
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,1023,0.04574933151404063
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,float16,31,0.012154666086037954
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,7,0.014912000546852747
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,2047,0.15196266770362854
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,31,0.014794666320085526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,2047,0.08269333342711131
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,float16,4095,0.29769599437713623
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,1,0.010757333288590113
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,4095,0.15634133418401083
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,1,0.00877333308259646
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,3,0.010890666395425797
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,7,0.010677333921194077
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,3,0.008997333546479544
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,7,0.009093333035707474
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,15,0.010816000401973724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,15,0.008965333302815756
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,31,0.00902399979531765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,63,0.01071999967098236
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,127,0.010693332801262537
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,127,0.008863999818762144
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,255,0.010794666906197866
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,255,0.00891733355820179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,511,0.012879999975363413
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,511,0.01129066695769628
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,1023,0.02827200045188268
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,1023,0.015087999403476715
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,1,0.012858666479587555
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,1,0.01071999967098236
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,float16,31,0.01062400018175443
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,3,0.012815999488035837
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,3,0.010853332777818045
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,7,0.012789333860079447
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,7,0.010837333897749582
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,15,0.012736000120639801
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,63,0.00895999992887179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,15,0.010751999914646149
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,31,0.012730666746695837
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,31,0.010640000303586325
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,63,0.012752000242471695
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,63,0.010773333410422007
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,127,0.010885333021481832
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,127,0.012890666723251343
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,255,0.011893333246310553
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,255,0.010821333775917688
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,511,0.027029333015282948
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,511,0.014032000054915747
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,float16,1023,0.04602666695912679
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,1023,0.028064000109831493
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,1,0.016917333006858826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,1,0.014981333166360855
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,3,0.016751999656359356
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,3,0.014949332922697067
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,7,0.015018666783968607
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,31,0.017269333203633625
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,31,0.015210667004187902
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,63,0.0170666662355264
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,63,0.014778666198253632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,127,0.01716800034046173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,15,0.014858666807413101
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,127,0.014837333311637243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,255,0.014746667196353277
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,255,0.026352000733216602
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,511,0.04516266783078512
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,511,0.02754666656255722
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,7,0.016885332763195038
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,1,0.027114666998386383
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,1023,0.08092799782752991
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,1023,0.046207999189694725
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,3,0.027130665878454845
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,7,0.02701866626739502
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,3,0.023034666975339253
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,float16,15,0.01717866708834966
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,7,0.022954667607943218
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,31,0.027503999571005504
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,31,0.02290133386850357
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,63,0.02749866743882497
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,127,0.02903999884923299
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,1,0.023258666197458904
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,127,0.023221333821614582
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,15,0.0230880007147789
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,15,0.027450665831565857
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,255,0.045514668027559914
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,255,0.028890666862328846
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,1023,0.08222933113574982
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,1023,0.15261333187421164
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,float16,511,0.08125333487987518
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,63,0.02310933421055476
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,1,0.014975999792416891
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,511,0.04572266836961111
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,1,0.012671999633312225
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,3,0.01479999969402949
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,3,0.012837332983811697
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,7,0.014943999548753103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,15,0.01292266696691513
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,15,0.014959999670584997
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,7,0.01268799975514412
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,31,0.012618667135636011
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,63,0.014677333335081736
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,63,0.012794667234023413
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,31,0.014938666174809137
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,127,0.014757333944241205
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,127,0.012800000607967377
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,255,0.014762666076421738
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,255,0.012725333372751871
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,float16,511,0.02759466568628947
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,1,0.01482133318980535
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,1,0.017231999586025875
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,511,0.01618133361140887
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,3,0.016949333250522614
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,3,0.014965333044528961
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,7,0.017114666601022083
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,7,0.014837333311637243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,15,0.01708799973130226
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,15,0.015109332899252573
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,31,0.016864000509182613
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,31,0.014767999450365702
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,63,0.01684800038735072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,63,0.014826666563749313
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,127,0.01711999997496605
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,127,0.014922666052977243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,255,0.02703999976317088
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,255,0.014805333067973455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,1,0.027061333258946735
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,float16,511,0.045738667249679565
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,511,0.028399998943010967
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,1,0.02310933421055476
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,3,0.02716800073782603
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,3,0.02319466571013133
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,7,0.02741866558790207
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,15,0.02808533360560735
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,31,0.023077333966890972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,7,0.023445333043734234
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,63,0.027893332143624622
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,63,0.022853332261244457
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,15,0.023226665953795116
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,31,0.027280000348885853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,127,0.029365333418051403
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,127,0.02311466634273529
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,1,0.04781333108743032
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,1,0.039461334546407066
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,255,0.04585599899291992
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,255,0.02850666642189026
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,float16,511,0.08195733527342479
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,511,0.046053335070610046
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,3,0.04956266780694326
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,3,0.03973866750796636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,7,0.04785066843032837
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,7,0.0393653338154157
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,63,0.0491839994986852
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,15,0.04865066707134247
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,63,0.039359999199708305
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,127,0.04956800242265066
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,15,0.039664000272750854
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,1,0.007231999809543292
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,1,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,3,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,3,0.006949333474040031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,7,0.007167999943097432
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,7,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,15,0.007018666714429855
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,15,0.0069759997228781385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,255,0.08228266735871632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,31,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,63,0.007087999954819679
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,255,0.04841599861780802
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,63,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,127,0.04131733377774557
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,127,0.006954666847983996
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,31,0.00701333334048589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,255,0.006778666749596596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,511,0.008687999720374743
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,511,0.008912000184257826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,1023,0.008938666433095932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,1023,0.009002666920423508
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,2047,0.008885333314538002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,511,0.08275199929873149
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,2047,0.008912000184257826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,127,0.007125333572427432
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,31,0.048245335618654885
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,4095,0.008997333546479544
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,8191,0.012975999464591345
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,8191,0.012800000607967377
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,16383,0.015354666858911514
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,32767,0.031109333038330078
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,16383,0.014874666929244995
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,32767,0.01720533271630605
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,65535,0.049973333875338234
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,1,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,1,0.0069973332186539965
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,3,0.007178666690985362
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,3,0.006640000268816948
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,7,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,7,0.006735999758044879
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,15,0.006682666639486949
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,15,0.007093333328763644
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,31,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,31,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,63,0.006810666372378667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,4095,0.009072000160813332
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,63,0.006591999903321266
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,127,0.007120000198483467
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,127,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,255,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,255,0.007120000198483467
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,511,0.008650666723648706
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,511,0.00701333334048589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,1023,0.00892800030608972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,1023,0.008618666479984919
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,2047,0.010090666512648264
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,2047,0.008767999708652496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,4095,0.010933333386977514
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,8191,0.014277332772811254
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,8191,0.012736000120639801
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,float16,511,0.15416000286738077
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,16383,0.02938666691382726
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,16383,0.015034666905800501
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,31,0.039749334255854286
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,32767,0.02923733244339625
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,65535,0.08433066805203755
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,1,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,65535,0.04844266672929128
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,65535,0.03153600047032038
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,3,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,3,0.006634666894872983
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,7,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,7,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,15,0.006821333120266597
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,15,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,31,0.00679466687142849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,63,0.006720000257094701
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,32767,0.04775999983151754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,63,0.006522666662931442
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,127,0.006954666847983996
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,255,0.006842666616042455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,127,0.006842666616042455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,255,0.006730666384100914
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,511,0.008570666735370954
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,511,0.007104000076651573
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,1023,0.008693333094318708
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,1023,0.008602666358153025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,2047,0.010693332801262537
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,2047,0.008874666566650072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,float16,4095,0.010944000134865442
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,4095,0.012768000364303589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,8191,0.02720000098148982
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,8191,0.012714666624863943
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,16383,0.0444160004456838
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,16383,0.02741866558790207
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,float16,255,0.007087999954819679
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,32767,0.08086400230725606
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,32767,0.04606399933497111
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,1,0.006666666517655055
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,65535,0.08365866541862488
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,65535,0.1537920037905375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,3,0.006842666616042455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,1,0.006725333631038666
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,3,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,7,0.006640000268816948
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,7,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,15,0.006746666505932808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,15,0.006789333497484525
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,31,0.007007999966541926
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,31,0.006746666505932808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,63,0.006746666505932808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,63,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,127,0.006602666651209195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,127,0.006768000001708667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,float16,31,0.006746666505932808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,255,0.006634666894872983
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,255,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,511,0.008752000207702318
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,511,0.008336000144481659
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,1023,0.008943999807039896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,2047,0.010672000547250112
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,2047,0.011087999989589056
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,4095,0.026629333694775898
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,8191,0.04465599854787191
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,16383,0.08029333253701527
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,16383,0.04555733501911163
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,4095,0.010949333508809408
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,32767,0.1520853340625763
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,32767,0.08117333551247914
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,65535,0.29660799105962116
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,65535,0.15350932876269022
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,float16,1,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,1023,0.008965333302815756
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,4095,0.013034666577974955
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,8191,0.027024000883102417
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,1,0.01685333376129468
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,1,0.021781332790851593
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,3,0.022272000710169475
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,3,0.01691199963291486
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,7,0.021941334009170532
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,15,0.022890667120615642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,7,0.01721599946419398
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,15,0.016890666137139004
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,31,0.02186666677395503
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,31,0.01717866708834966
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,63,0.021530665457248688
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,63,0.01685333376129468
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,127,0.02218666672706604
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,127,0.01700266698996226
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,float16,255,0.02924799919128418
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,255,0.016997333616018295
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,1,0.027461332579453785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,1,0.022944000860055287
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,3,0.027450665831565857
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,3,0.023007998863856
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,7,0.02717333287000656
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,15,0.02749866743882497
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,7,0.02327466756105423
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,15,0.023269332945346832
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,31,0.02716800073782603
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,31,0.023002666731675465
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,63,0.02734400083621343
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,63,0.023050665855407715
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,127,0.030741333961486816
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,127,0.023002666731675465
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,1,0.0396373321612676
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,1,0.047637333472569786
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,255,0.029232000311215717
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,float16,255,0.04571733375390371
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,3,0.048165331284205117
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,3,0.03956266740957896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,7,0.04953599969546
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,7,0.039408000806967415
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,15,0.0476746658484141
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,15,0.03944533318281174
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,31,0.04806933303674062
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,63,0.048986668388048805
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,63,0.039664000272750854
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,31,0.039664000272750854
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,127,0.05177066723505656
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,127,0.04048000027736028
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,1,0.08782933155695598
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,1,0.07229333122571309
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,float16,255,0.08267733454704285
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,255,0.0491893341143926
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,3,0.08828799923261006
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,3,0.07262933254241943
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,7,0.08869866530100505
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,15,0.08842666943868001
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,15,0.0724426656961441
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,7,0.07261866827805837
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,63,0.08878399928410848
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,63,0.07228800157705943
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,31,0.08880533774693807
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,31,0.07259733478228252
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,127,0.09214400251706441
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,127,0.07211733361085255
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,float16,255,0.156031996011734
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,255,0.0881119966506958
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,1,0.03764266769091288
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,1,0.02736533433198929
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,3,0.037263999382654824
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,3,0.02739199995994568
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,7,0.03745600084463755
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,7,0.02736533433198929
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,31,0.03783999880154928
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,15,0.037685332198937736
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,15,0.02749866743882497
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,31,0.027274665733178455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,63,0.03764266769091288
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,63,0.027242665489514668
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,float16,127,0.03869866579771042
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,127,0.027306665976842243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,1,0.04855999847253164
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,1,0.039450667798519135
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,3,0.04830400149027506
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,7,0.04862933357556661
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,3,0.0396373321612676
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,7,0.03966933240493139
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,15,0.03961600114901861
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,15,0.0498986691236496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,31,0.0396373321612676
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,63,0.04880533119042715
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,63,0.03962666789690653
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,127,0.04141333450873693
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,127,0.05218133330345154
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,float16,31,0.04857600231965383
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,1,0.08884800473848979
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,1,0.07264000177383423
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,3,0.08877866466840108
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,3,0.07250133156776428
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,7,0.08888000249862671
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,15,0.07223466535409291
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,15,0.0890880028406779
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,31,0.08888000249862671
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,31,0.07234133283297221
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,7,0.07227733234564464
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,63,0.08768000205357869
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,63,0.07213333249092102
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,float16,127,0.09288000067075093
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,1,0.16726932922999063
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,127,0.0734506646792094
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,1,0.1369653344154358
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,3,0.1358453333377838
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,3,0.16802666584650675
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,7,0.16725333531697592
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,7,0.1359946628411611
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,15,0.1676373283068339
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,1,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,15,0.13589866956075033
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,1,0.00679466687142849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,31,0.16704533497492471
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,3,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,7,0.007050666958093643
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,15,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,3,0.006831999868154526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,15,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,31,0.007029333462317784
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,31,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,7,0.00666133314371109
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,63,0.007135999699433644
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,31,0.1358453333377838
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,63,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,127,0.0069759997228781385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,127,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,255,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,511,0.00860799973209699
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,511,0.00898133342464765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,1023,0.00980266680320104
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,1023,0.008656000097592672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,2047,0.008922666932145754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,2047,0.00891733355820179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,255,0.00702400008837382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,4095,0.012655999511480331
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,4095,0.011002667248249054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,8191,0.015029333531856537
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,8191,0.012847999731699625
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,16383,0.029178666571776073
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,16383,0.017103999853134155
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,float16,32767,0.04823466638724009
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,1,0.006778666749596596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,1,0.0069866664707660675
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,3,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,3,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,7,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,7,0.006560000280539195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,15,0.006842666616042455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,15,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,31,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,31,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,63,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,63,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,127,0.006842666616042455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,63,0.16854933897654215
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,255,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,255,0.006805333619316419
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,511,0.00854399986565113
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,1023,0.00873066671192646
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,1023,0.008938666433095932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,2047,0.010672000547250112
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,2047,0.008992000172535578
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,4095,0.012858666479587555
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,511,0.007621333623925845
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,8191,0.027295999228954315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,8191,0.013088000317414602
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,16383,0.045552000403404236
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,32767,0.03105599929889043
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,16383,0.027376001079877216
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,32767,0.04605866471926371
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,float16,32767,0.08252266546090443
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,1,0.007167999943097432
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,float16,127,0.17450666427612305
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,1,0.006528000036875407
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,3,0.006735999758044879
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,3,0.007376000285148621
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,7,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,15,0.0069440001000960665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,15,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,31,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,127,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,31,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,63,0.007231999809543292
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,63,0.006720000257094701
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,127,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,127,0.006810666372378667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,127,0.13701333602269491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,511,0.009162666896979014
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,511,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,1023,0.00902399979531765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,2047,0.011477333803971609
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,4095,0.01098666712641716
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,2047,0.010714666297038397
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,7,0.0069919998447100324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,4095,0.013002666334311167
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,8191,0.0458186666170756
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,8191,0.027232001225153606
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,16383,0.045519997676213585
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,16383,0.0804746647675832
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,32767,0.15333867073059082
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,63,0.1356000006198883
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,1023,0.008559999987483025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,32767,0.0825973351796468
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,1,0.0069440001000960665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,3,0.0069919998447100324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,3,0.008767999708652496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,7,0.006805333619316419
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,15,0.007882666463653246
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,255,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,7,0.007717333113153775
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,31,0.008597333605090777
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,31,0.006949333474040031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,63,0.006624000146985054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,127,0.00679466687142849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,float16,4095,0.027098665634791057
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,255,0.008581333483258883
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,255,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,511,0.00878399983048439
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,511,0.00854399986565113
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,1,0.007333333293596904
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,1023,0.008997333546479544
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,2047,0.025370667378107708
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,2047,0.012789333860079447
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,4095,0.025968000292778015
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,255,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,8191,0.044480000933011375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,63,0.0086666668454806
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,16383,0.151936004559199
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,127,0.008949333180983862
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,1,0.007120000198483467
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,32767,0.29606932401657104
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,32767,0.1525706648826599
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,1,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,3,0.007040000210205714
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,3,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,7,0.007221333061655362
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,7,0.006714666883150737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,15,0.0069866664707660675
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,15,0.019621333728233974
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,31,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,31,0.006805333619316419
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,63,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,63,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,127,0.006720000257094701
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,127,0.007365333537260692
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,255,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,255,0.006629333520929019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,15,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,511,0.00902399979531765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,511,0.008576000109314919
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,1023,0.009029333169261614
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,8191,0.08057066798210144
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,1023,0.00895999992887179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,2047,0.01090666651725769
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,4095,0.01293333371480306
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,8191,0.02758399893840154
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,4095,0.010826667149861654
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,8191,0.01498666654030482
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,16383,0.08035733302434285
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,float16,16383,0.04633066554864248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,16383,0.02926933268706004
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,1,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,3,0.006666666517655055
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,7,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,7,0.006821333120266597
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,15,0.006597333277265231
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,15,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,31,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,31,0.006698666761318843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,1023,0.010650667051474253
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,63,0.007002666592597961
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,127,0.006746666505932808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,127,0.006570666407545407
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,255,0.007071999832987785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,255,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,511,0.008613333106040955
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,511,0.008682666967312494
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,float16,4095,0.04375466704368591
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,1023,0.009599999835093817
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,1023,0.008639999975760778
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,2047,0.011152000476916632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,4095,0.027221334477265675
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,2047,0.010714666297038397
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,4095,0.01310933381319046
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,2047,0.008986666798591614
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,8191,0.027029333015282948
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,8191,0.045519997676213585
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,16383,0.04577599962552389
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,1,0.00867733359336853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,16383,0.08167466521263123
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,1,0.006810666372378667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,3,0.008767999708652496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,3,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,7,0.008767999708652496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,1,0.0069759997228781385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,7,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,15,0.008778666456540426
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,15,0.00706666645904382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,31,0.00873066671192646
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,31,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,63,0.006768000001708667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,127,0.0085333331177632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,127,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,255,0.008623999853928884
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,255,0.006714666883150737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,511,0.008943999807039896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,63,0.006821333120266597
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,1023,0.010832000523805618
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,1023,0.008821333448092142
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,2047,0.013034666577974955
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,2047,0.025562666356563568
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,4095,0.027098665634791057
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,8191,0.04475200176239014
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,8191,0.08086400230725606
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,1,0.007055999711155891
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,16383,0.1529759963353475
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,16383,0.08242133259773254
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,3,0.006704000135262807
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,3,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,7,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,7,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,15,0.006762666627764702
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,15,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,31,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,31,0.006720000257094701
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,float16,3,0.006757333253820737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,63,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,127,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,127,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,63,0.00966933307548364
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,255,0.006714666883150737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,511,0.007258666679263115
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,1023,0.008672000219424566
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,511,0.008576000109314919
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,1023,0.00867733359336853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,2047,0.009061333412925402
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,2047,0.008581333483258883
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,4095,0.008997333546479544
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,4095,0.008703999842206636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,8191,0.010597333312034607
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,8191,0.008986666798591614
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,16383,0.014688000082969666
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,16383,0.012725333372751871
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,32767,0.01710933322707812
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,32767,0.016986666868130367
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,65535,0.025061334172884624
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,65535,0.024906667570273083
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,1,0.006831999868154526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,1,0.006768000001708667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,3,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,3,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,7,0.006618666773041089
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,7,0.0064853330453236895
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,15,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,15,0.006586666529377301
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,31,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,31,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,63,0.006624000146985054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,63,0.006671999891599019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,127,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,127,0.006810666372378667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,255,0.007151999821265538
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,255,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,511,0.008682666967312494
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,63,0.007087999954819679
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,1023,0.008842666943868002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,1023,0.006831999868154526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,255,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,2047,0.008767999708652496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,2047,0.007871999715765318
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,4095,0.008618666479984919
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,4095,0.008752000207702318
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,8191,0.009546666716535887
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,8191,0.008949333180983862
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,16383,0.013007999708255133
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,16383,0.012842666357755661
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,32767,0.01743999992807706
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,32767,0.017210666090250015
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,65535,0.019280000279347103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,float16,65535,0.021290667355060577
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,1,0.006981333096822103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,1,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,3,0.006624000146985054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,7,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,7,0.006581333155433337
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,15,0.006634666894872983
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,15,0.006751999879876773
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,31,0.006751999879876773
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,31,0.006624000146985054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,63,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,63,0.006624000146985054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,127,0.007007999966541926
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,127,0.006949333474040031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,1,0.006645333642760913
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,255,0.006741333131988843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,255,0.006640000268816948
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,511,0.007002666592597961
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,1023,0.008858666444818178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,1023,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,2047,0.0086666668454806
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,4095,0.008597333605090777
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,4095,0.008810666700204214
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,8191,0.009279999881982803
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,8191,0.008949333180983862
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,511,0.006805333619316419
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,16383,0.013450667262077332
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,16383,0.012810666114091873
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,32767,0.016949333250522614
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,float16,511,0.008682666967312494
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,65535,0.03324266771475474
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,65535,0.017237332959969837
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,1,0.00707733320693175
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,1,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,3,0.006784000123540561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,3,0.006746666505932808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,7,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,7,0.006981333096822103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,float16,4095,0.04419200122356415
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,15,0.006650666395823161
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,31,0.00706666645904382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,31,0.006725333631038666
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,63,0.00702400008837382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,63,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,127,0.0069919998447100324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,511,0.008709333216150602
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,255,0.006831999868154526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,255,0.006981333096822103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,511,0.008752000207702318
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,float16,2047,0.008592000231146812
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,511,0.008858666444818178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,1023,0.009152000149091085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,1023,0.008778666456540426
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,2047,0.009056000038981438
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,2047,0.008810666700204214
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,4095,0.008879999940594038
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,4095,0.008954666554927826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,8191,0.011002667248249054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,8191,0.00903466654320558
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,16383,0.01368533323208491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,16383,0.012938667088747025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,32767,0.01894933357834816
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,32767,0.017781333376963932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,65535,0.021562665700912476
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,65535,0.020581333587567013
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,32767,0.01545599972208341
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,1,0.00721066693464915
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,1,0.006741333131988843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,3,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,3,0.00655466690659523
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,7,0.006954666847983996
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,7,0.006714666883150737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,15,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,15,0.006693333387374878
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,31,0.007029333462317784
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,31,0.00666133314371109
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,63,0.006698666761318843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,3,0.006565333033601443
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,127,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,127,0.007029333462317784
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,255,0.006784000123540561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,255,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,511,0.00744000015159448
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,float16,15,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,1023,0.00707733320693175
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,1023,0.008949333180983862
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,2047,0.009088000282645226
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,2047,0.008821333448092142
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,4095,0.0086666668454806
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,4095,0.008517333616813024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,8191,0.010640000303586325
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,8191,0.009045333291093508
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,16383,0.014186666657527288
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,16383,0.012890666723251343
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,32767,0.01724799970785777
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,32767,0.015082667271296183
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,65535,0.033733333150545754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,65535,0.017301333447297413
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,127,0.006704000135262807
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,1,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,1,0.006677333265542984
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,3,0.00701333334048589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,3,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,7,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,7,0.006698666761318843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,15,0.007007999966541926
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,31,0.007151999821265538
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,63,0.006655999769767125
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,127,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,127,0.006735999758044879
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,255,0.006586666529377301
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,255,0.00702400008837382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,511,0.008698666468262672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,511,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,1023,0.00891733355820179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,1023,0.008282666405042013
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,2047,0.008746666833758354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,2047,0.008597333605090777
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,4095,0.00892800030608972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,4095,0.008767999708652496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,8191,0.01240533341964086
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,8191,0.011237333218256632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,16383,0.01479999969402949
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,16383,0.013066666821638743
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,63,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,32767,0.030693332354227703
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,32767,0.015813333292802174
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,65535,0.04889066517353058
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,65535,0.030410667260487873
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,float16,511,0.008874666566650072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,1,0.007285333548982938
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,1,0.006810666372378667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,3,0.0069973332186539965
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,3,0.00701333334048589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,7,0.007109333450595538
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,7,0.006805333619316419
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,15,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,15,0.0069973332186539965
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,31,0.007429333403706551
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,31,0.006954666847983996
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,15,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,31,0.006671999891599019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,float16,63,0.007189333438873291
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,63,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,127,0.007525333513816197
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,127,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,255,0.006831999868154526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,511,0.008576000109314919
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,511,0.009189333145817121
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,1023,0.011215999722480774
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,1023,0.008954666554927826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,2047,0.012752000242471695
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,2047,0.010885333021481832
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,8191,0.04554133117198944
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,4095,0.02739733209212621
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,4095,0.015066667149464289
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,8191,0.02794133375088374
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,1,0.0086666668454806
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,1,0.007018666714429855
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,3,0.008746666833758354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,7,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,3,0.007120000198483467
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,15,0.008762666955590248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,15,0.007130666946371396
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,31,0.008458666503429413
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,31,0.00702400008837382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,63,0.006768000001708667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,127,0.006709333509206772
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,127,0.008890666688481966
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,255,0.008618666479984919
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,63,0.006778666749596596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,255,0.007040000210205714
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,511,0.00897066667675972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,float16,255,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,511,0.00884799969693025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,1023,0.010981333752473196
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,1023,0.009183999771873156
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,4095,0.044112001856168113
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,2047,0.026288000245889027
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,2047,0.012917333592971167
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,4095,0.027104000250498455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,8191,0.08191466828187306
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,8191,0.045279999574025474
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,1,0.008896000062425932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,3,0.008826666822036108
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,7,0.008986666798591614
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,3,0.008992000172535578
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,15,0.00898133342464765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,15,0.008863999818762144
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,7,0.008752000207702318
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,31,0.008570666735370954
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,63,0.008949333180983862
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,63,0.008725333337982496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,127,0.008826666822036108
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,127,0.008703999842206636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,255,0.008943999807039896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,255,0.008703999842206636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,511,0.01099733387430509
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,511,0.010698666175206503
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,1023,0.025914666553338368
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,1023,0.012928000340859095
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,2047,0.02734400083621343
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,4095,0.0803306649128596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,4095,0.04584000011285146
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,8191,0.15398933490117392
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,1,0.007007999966541926
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,3,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,1,0.006591999903321266
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,7,0.008757333581646284
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,3,0.007018666714429855
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,7,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,7,0.006640000268816948
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,15,0.007050666958093643
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,15,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,31,0.006735999758044879
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,31,0.00897066667675972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,31,0.00706666645904382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,63,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,63,0.006560000280539195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,127,0.007082666580875714
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,127,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,255,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,255,0.007071999832987785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,float16,63,0.008634666601816813
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,511,0.009002666920423508
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,511,0.007920000081261
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,1023,0.00860799973209699
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,1023,0.008656000097592672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,2047,0.00878399983048439
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,2047,0.008885333314538002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,4095,0.008901333436369896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,8191,0.010938666760921478
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,8191,0.00897066667675972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,16383,0.01545599972208341
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,16383,0.014192000031471252
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,32767,0.0183999997874101
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,32767,0.0170666662355264
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,65535,0.033615998923778534
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,65535,0.01929066702723503
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,1,0.0069440001000960665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,1,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,3,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,3,0.006560000280539195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,float16,2047,0.04366933306058248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,7,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,7,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,15,0.007114666824539502
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,15,0.006810666372378667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,31,0.007018666714429855
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,31,0.00702400008837382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,63,0.006560000280539195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,63,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,127,0.0069440001000960665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,127,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,255,0.006789333497484525
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,255,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,511,0.008703999842206636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,511,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,1023,0.008858666444818178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,1023,0.008592000231146812
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,2047,0.008725333337982496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,2047,0.008832000195980072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,1,0.008837333569924036
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,4095,0.00891733355820179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,8191,0.08374933401743571
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,8191,0.012133333832025528
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,8191,0.010922666639089584
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,16383,0.01309866706530253
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,16383,0.014981333166360855
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,32767,0.031034665803114574
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,32767,0.015594666202863058
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,float16,65535,0.048528000712394714
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,65535,0.03086400032043457
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,1,0.006735999758044879
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,1,0.006480000292261441
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,3,0.006949333474040031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,3,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,7,0.006730666384100914
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,7,0.0068800002336502075
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,15,0.00660800002515316
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,15,0.006704000135262807
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,31,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,63,0.006810666372378667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,127,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,127,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,255,0.006831999868154526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,255,0.006640000268816948
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,511,0.00892800030608972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,511,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,1023,0.008826666822036108
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,1023,0.008672000219424566
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,2047,0.008549333239595095
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,4095,0.010863999525705973
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,4095,0.009999999776482582
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,8191,0.014736000448465347
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,8191,0.012869333227475485
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,16383,0.029093332588672638
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,16383,0.014837333311637243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,32767,0.047295997540156044
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,32767,0.029450667401154835
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,65535,0.08268799881140391
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,65535,0.04804799954096476
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,4095,0.008661333471536636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,1,0.008570666735370954
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,1,0.008570666735370954
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,3,0.008821333448092142
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,3,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,7,0.008517333616813024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,31,0.006597333277265231
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,63,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,15,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,31,0.008586666857202848
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,31,0.008698666468262672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,63,0.008682666967312494
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,float16,2047,0.009045333291093508
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,63,0.007018666714429855
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,127,0.008789333204428354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,127,0.008672000219424566
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,255,0.008736000085870424
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,255,0.008709333216150602
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,511,0.008816000074148178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,511,0.008890666688481966
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,1023,0.01201066623131434
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,1023,0.010741333166758219
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,2047,0.02735466758410136
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,2047,0.01302933320403099
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,1,0.009018666421373686
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,7,0.008645333349704742
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,4095,0.045509333411852516
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,4095,0.027466667195161183
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,1,0.008943999807039896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,float16,15,0.008736000085870424
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,3,0.008986666798591614
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,3,0.008709333216150602
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,15,0.00871999996403853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,15,0.008789333204428354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,7,0.00891733355820179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,31,0.00895999992887179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,7,0.00898133342464765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,63,0.008965333302815756
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,63,0.008746666833758354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,127,0.008997333546479544
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,255,0.009109333157539368
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,255,0.008634666601816813
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,511,0.012250666817029318
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,511,0.010751999914646149
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,1023,0.02714666724205017
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,1023,0.014021333307027817
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,float16,4095,0.008933333059151968
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,2047,0.04587199787298838
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,2047,0.02733866622050603
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,4095,0.04572799801826477
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,1,0.012367999802033106
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,1,0.010885333021481832
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,31,0.008576000109314919
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,7,0.010944000134865442
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,7,0.011445333560307821
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,15,0.012639999389648438
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,127,0.008922666932145754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,3,0.012960000584522883
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,3,0.01089599976936976
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,31,0.012650666137536367
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,31,0.010992000500361124
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,63,0.01268799975514412
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,63,0.010741333166758219
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,127,0.012938667088747025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,255,0.012341332932313284
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,255,0.01099733387430509
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,511,0.027029333015282948
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,511,0.013397333522637686
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,float16,4095,0.08257066706816356
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,1023,0.045552000403404236
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,1023,0.027429332335789997
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,2047,0.04618666569391886
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,4095,0.08348266283671062
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,4095,0.15612799922625223
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,15,0.011663999408483505
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,1,0.010869332899649939
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,3,0.010842667271693548
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,1,0.009072000160813332
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,3,0.008922666932145754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,7,0.010762666662534079
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,127,0.010714666297038397
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,7,0.009056000038981438
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,15,0.010784000158309937
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,15,0.008687999720374743
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,31,0.011018666128317514
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,31,0.00927466650803884
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,63,0.010746666540702185
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,63,0.009066666786869368
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,127,0.010847999403874079
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,127,0.008789333204428354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,255,0.010714666297038397
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,255,0.009114666531483332
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,511,0.013194666554530462
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,511,0.011125333607196808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,float16,1023,0.02720533311367035
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,1,0.012810666114091873
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,1023,0.014815999815861383
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,1,0.010911999891201654
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,3,0.012725333372751871
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,3,0.010949333508809408
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,7,0.012789333860079447
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,7,0.010901333143313726
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,15,0.012602667013804117
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,15,0.01073066641887029
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,31,0.012858666479587555
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,31,0.010858666151762009
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,63,0.012613333761692047
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,63,0.011050666371981302
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,127,0.0129120002190272
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,127,0.01071999967098236
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,255,0.012624000509579977
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,255,0.010661333799362183
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,511,0.027295999228954315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,511,0.012986666212479273
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,float16,1023,0.04557333389918009
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,1,0.016842667013406754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,1023,0.027280000348885853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,1,0.014789332946141561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,3,0.016858667135238647
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,3,0.01479999969402949
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,7,0.01729600007335345
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,31,0.017050666113694508
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,15,0.01695466662446658
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,7,0.01552533358335495
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,63,0.017029333859682083
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,15,0.014752000570297241
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,float16,2047,0.08242666721343994
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,127,0.017071999609470367
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,127,0.014783999572197596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,255,0.02700799951950709
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,255,0.014991999914248785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,511,0.045408000548680626
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,511,0.02731200059254964
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,31,0.014805333067973455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,float16,1023,0.08045333127180736
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,1023,0.04655466477076212
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,1,0.014725333700577417
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,63,0.014661333213249842
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,1,0.012949333836634954
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,3,0.01498666654030482
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,3,0.012890666723251343
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,15,0.014666666587193808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,15,0.012703999876976013
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,31,0.015050667027632395
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,7,0.012842666357755661
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,7,0.01482133318980535
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,31,0.012869333227475485
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,63,0.014655999839305878
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,63,0.012901333471139273
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,127,0.014655999839305878
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,127,0.012634667257467905
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,255,0.015066667149464289
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,255,0.012831999609867731
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,1,0.01709866647919019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,float16,511,0.027471999327341717
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,511,0.015098666151364645
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,1,0.014981333166360855
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,3,0.01704000060757001
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,3,0.014837333311637243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,7,0.014949332922697067
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,15,0.01682666689157486
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,7,0.017125333348910015
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,15,0.014789332946141561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,31,0.015029333531856537
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,63,0.01681600014368693
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,63,0.014629332969586054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,127,0.016858667135238647
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,127,0.015125333021084467
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,255,0.02734400083621343
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,1,0.027248000105222065
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,255,0.01471466695268949
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,511,0.027349332968393963
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,511,0.045696000258127846
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,1,0.02314666658639908
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,float16,31,0.01718933383623759
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,3,0.027072000006834667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,7,0.027093333502610523
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,3,0.023242667317390442
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,7,0.022954667607943218
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,15,0.02735999971628189
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,31,0.027450665831565857
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,31,0.022970666488011677
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,63,0.027456000447273254
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,15,0.02329600105683009
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,63,0.02313599983851115
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,1,0.006704000135262807
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,3,0.006805333619316419
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,127,0.02292799949645996
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,127,0.029418667157491047
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,3,0.006800000245372455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,15,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,15,0.006704000135262807
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,7,0.007040000210205714
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,31,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,255,0.045519997676213585
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,31,0.006666666517655055
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,63,0.0069973332186539965
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,63,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,127,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,255,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,127,0.0069759997228781385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,511,0.008885333314538002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,float16,511,0.08133333424727122
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,511,0.008645333349704742
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,1023,0.008693333094318708
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,511,0.045909335215886436
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,1023,0.008885333314538002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,2047,0.009002666920423508
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,4095,0.009029333169261614
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,8191,0.012975999464591345
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,1,0.0069759997228781385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,8191,0.012709333250919977
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,16383,0.015295999745527903
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,16383,0.014970666418472925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,32767,0.031301334500312805
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,32767,0.017530667285124462
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,65535,0.04957866668701172
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,1,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,2047,0.00902399979531765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,float16,4095,0.008965333302815756
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,1,0.00666133314371109
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,3,0.006704000135262807
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,3,0.006671999891599019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,7,0.006831999868154526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,7,0.006624000146985054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,15,0.0069866664707660675
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,15,0.00666133314371109
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,31,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,31,0.006640000268816948
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,7,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,63,0.006591999903321266
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,63,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,127,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,127,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,255,0.006575999781489372
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,255,0.006714666883150737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,255,0.027866666515668232
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,511,0.006800000245372455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,1023,0.008757333581646284
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,2047,0.008901333436369896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,255,0.0069440001000960665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,4095,0.011087999989589056
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,2047,0.008976000050703684
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,8191,0.012842666357755661
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,16383,0.014933332800865173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,32767,0.047541335225105286
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,32767,0.02961066613594691
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,65535,0.03129599988460541
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,65535,0.08436266581217448
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,1,0.00679466687142849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,1,0.006730666384100914
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,3,0.007002666592597961
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,3,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,511,0.00860799973209699
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,1023,0.008592000231146812
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,7,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,15,0.0069866664707660675
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,15,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,31,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,31,0.006677333265542984
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,4095,0.010879999647537867
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,63,0.006629333520929019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,63,0.006597333277265231
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,127,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,127,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,16383,0.02934933453798294
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,255,0.006741333131988843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,511,0.008757333581646284
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,1023,0.008767999708652496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,1023,0.00903466654320558
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,2047,0.01080000028014183
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,2047,0.009103999783595404
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,4095,0.012890666723251343
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,4095,0.010784000158309937
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,8191,0.02718399961789449
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,8191,0.012975999464591345
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,16383,0.04549333453178406
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,16383,0.027482666075229645
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,65535,0.04855999847253164
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,32767,0.08055999875068665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,32767,0.04576000074545542
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,65535,0.15397333105405173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,65535,0.08467732866605122
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,7,0.006645333642760913
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,float16,8191,0.014058666924635569
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,float16,255,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,511,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,1,0.021759999295075733
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,1,0.017360000560681026
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,3,0.016869333883126576
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,3,0.022965334355831146
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,7,0.022757334013779957
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,7,0.016794666647911072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,15,0.022106667359670002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,15,0.017093333105246227
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,31,0.02195200075705846
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,31,0.017157333592573803
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,63,0.023007998863856
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,63,0.017152000218629837
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,127,0.01720533271630605
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,127,0.022805333137512207
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,float16,255,0.027066667874654133
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,255,0.01729600007335345
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,1,0.027109332382678986
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,1,0.023007998863856
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,3,0.027295999228954315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,3,0.02298133323589961
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,7,0.027466667195161183
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,7,0.023168000082174938
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,15,0.02741333345572154
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,31,0.027141332626342773
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,15,0.023285334308942158
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,31,0.023029332359631855
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,63,0.027269333600997925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,63,0.022965334355831146
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,127,0.02972800036271413
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,127,0.023061332603295643
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,float16,255,0.04607999821503957
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,255,0.02934933453798294
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,1,0.047797332207361855
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,1,0.03950933367013931
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,3,0.048058668772379555
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,3,0.039690665900707245
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,7,0.0489279975493749
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,7,0.03978666663169861
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,15,0.04955733319123586
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,15,0.03972266614437103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,31,0.03928533444801966
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,31,0.04886933167775472
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,63,0.04916266600290934
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,63,0.03942933430274328
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,127,0.050570666790008545
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,127,0.04138133426507314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,float16,255,0.0823466678460439
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,255,0.048170665899912514
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,1,0.03738133360942205
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,1,0.027434666951497395
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,3,0.03792533278465271
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,3,0.02703999976317088
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,7,0.027488000690937042
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,7,0.03777066618204117
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,15,0.03736000011364619
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,15,0.0273333340883255
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,31,0.03737599899371465
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,63,0.037632000943024956
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,31,0.02720533311367035
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,63,0.027162666122118633
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,float16,127,0.03796799977620443
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,127,0.02720533311367035
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,1,0.04769066472848257
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,1,0.03966933240493139
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,3,0.04844800134499868
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,3,0.03955200066169103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,7,0.04919999837875366
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,7,0.039642666776975
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,15,0.049551998575528465
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,31,0.048800001541773476
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,15,0.0397119993964831
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,31,0.03967999915281931
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,63,0.04865066707134247
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,63,0.03956266740957896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,127,0.04120533416668574
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,float16,127,0.05179733534653982
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,1,0.08879466851552327
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,1,0.07238399982452393
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,3,0.08850133419036865
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,3,0.07282666862010956
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,7,0.07342933118343353
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,7,0.08861333131790161
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,15,0.08850666880607605
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,1,0.006751999879876773
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,15,0.07237866520881653
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,1,0.006981333096822103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,31,0.0885653297106425
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,3,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,3,0.007045333584149678
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,31,0.07211199899514516
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,7,0.007189333438873291
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,15,0.007071999832987785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,31,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,15,0.006704000135262807
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,31,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,63,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,63,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,127,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,127,0.007071999832987785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,255,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,255,0.00666133314371109
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,511,0.00879466657837232
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,511,0.008592000231146812
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,1023,0.009045333291093508
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,1023,0.008832000195980072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,63,0.0888320008913676
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,2047,0.008949333180983862
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,2047,0.008538666491707167
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,4095,0.013013333082199097
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,4095,0.010981333752473196
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,8191,0.014912000546852747
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,16383,0.029674666623274486
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,16383,0.016842667013406754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,float16,32767,0.047824000318845115
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,8191,0.013242666920026144
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,32767,0.03149333347876867
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,1,0.006810666372378667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,1,0.006757333253820737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,3,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,3,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,7,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,7,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,15,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,31,0.006757333253820737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,31,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,63,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,127,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,7,0.00702400008837382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,127,0.007018666714429855
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,255,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,255,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,511,0.00898133342464765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,511,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,1023,0.008922666932145754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,1023,0.00892800030608972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,2047,0.010656000425418219
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,2047,0.008879999940594038
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,4095,0.012954667210578918
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,4095,0.010549332946538925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,8191,0.027136000494162243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,float16,127,0.09321066737174988
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,8191,0.01312000056107839
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,16383,0.045754666129748024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,16383,0.027130665878454845
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,127,0.0744053324063619
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,32767,0.04827199876308441
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,1,0.007034666836261749
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,1,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,3,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,7,0.006624000146985054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,15,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,15,0.006768000001708667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,float16,32767,0.08239999910195668
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,15,0.007018666714429855
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,31,0.006624000146985054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,31,0.006650666395823161
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,63,0.0068800002336502075
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,3,0.006784000123540561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,63,0.07235200206438701
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,63,0.006730666384100914
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,127,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,127,0.0069759997228781385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,255,0.006624000146985054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,511,0.008698666468262672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,511,0.00891733355820179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,1023,0.008752000207702318
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,2047,0.010768000036478043
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,4095,0.026874666412671406
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,4095,0.012981332838535309
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,8191,0.04554666578769684
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,8191,0.027285332481066387
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,16383,0.08058133224646251
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,16383,0.04554133117198944
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,32767,0.15333867073059082
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,1,0.007071999832987785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,32767,0.08272533118724823
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,7,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,1,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,3,0.00701333334048589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,3,0.006544000158707301
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,7,0.007162666569153468
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,63,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,7,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,15,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,15,0.006741333131988843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,31,0.00679466687142849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,31,0.006821333120266597
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,63,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,63,0.006618666773041089
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,127,0.00707733320693175
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,127,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,255,0.006693333387374878
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,511,0.008661333471536636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,511,0.00706666645904382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,255,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,1023,0.00895999992887179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,1023,0.008629333227872849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,2047,0.010666667173306147
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,2047,0.009258666386206945
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,8191,0.027056001126766205
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,4095,0.01109333336353302
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,4095,0.013056000073750814
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,8191,0.013066666821638743
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,16383,0.02775466690460841
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,16383,0.045893331368764244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,1,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,32767,0.082805335521698
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,32767,0.046896000703175865
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,3,0.007061333085099856
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,3,0.00666133314371109
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,7,0.006757333253820737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,15,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,31,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,63,0.006821333120266597
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,63,0.006735999758044879
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,127,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,float16,255,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,127,0.006629333520929019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,255,0.006831999868154526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,255,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,511,0.009008000294367472
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,1023,0.009461333354314169
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,511,0.00701333334048589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,1023,0.009679999823371569
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,1023,0.008938666433095932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,2047,0.011194666226704916
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,1,0.006560000280539195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,4095,0.02699200063943863
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,7,0.0068800002336502075
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,4095,0.013034666577974955
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,8191,0.026965332527955372
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,31,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,16383,0.04573333263397217
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,16383,0.08275733391443889
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,32767,0.1556000014146169
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,1,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,32767,0.08310933411121368
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,3,0.008549333239595095
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,3,0.006682666639486949
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,7,0.00867733359336853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,7,0.00702400008837382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,15,0.0084906667470932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,15,0.006981333096822103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,float16,2047,0.011029332876205444
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,31,0.008816000074148178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,31,0.007333333293596904
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,63,0.008693333094318708
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,2047,0.010586666564146677
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,127,0.0086666668454806
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,127,0.006693333387374878
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,15,0.00702400008837382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,float16,8191,0.045066664616266884
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,255,0.006842666616042455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,255,0.008559999987483025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,511,0.008954666554927826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,511,0.008858666444818178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,1023,0.00901333304742972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,2047,0.012784000486135483
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,1,0.008736000085870424
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,4095,0.04378133515516917
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,4095,0.025653332471847534
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,8191,0.04548266530036926
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,8191,0.08062399923801422
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,16383,0.08110400040944417
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,1,0.009722666814923286
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,32767,0.15705066919326782
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,32767,0.2968053420384725
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,3,0.009045333291093508
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,3,0.008826666822036108
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,7,0.008869333192706108
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,63,0.006709333509206772
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,15,0.008703999842206636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,15,0.009045333291093508
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,31,0.009088000282645226
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,31,0.00867733359336853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,63,0.008922666932145754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,63,0.008559999987483025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,2047,0.02571200082699458
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,127,0.009018666421373686
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,255,0.008901333436369896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,255,0.00855466661353906
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,511,0.012768000364303589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,511,0.01080000028014183
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,16383,0.15321066975593567
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,1023,0.025418666501839954
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,2047,0.04377600053946177
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,2047,0.02698666602373123
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,4095,0.08022400240103404
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,4095,0.04554666578769684
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,7,0.009050666665037474
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,8191,0.08189333478609721
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,16383,0.2998773256937663
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,16383,0.15600533286730447
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,1,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,float16,1023,0.011045332998037338
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,127,0.008853333070874214
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,1,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,32767,0.5897866487503052
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,3,0.006602666651209195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,7,0.0069973332186539965
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,3,0.006586666529377301
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,15,0.007007999966541926
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,31,0.006549333532651265
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,7,0.006629333520929019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,31,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,63,0.006613333399097125
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,63,0.006575999781489372
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,127,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,127,0.006597333277265231
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,255,0.0068800002336502075
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,1023,0.012938667088747025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,255,0.0069440001000960665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,511,0.008698666468262672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,511,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,1023,0.00860799973209699
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,1023,0.008682666967312494
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,1,0.008869333192706108
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,2047,0.008837333569924036
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,4095,0.008997333546479544
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,4095,0.008901333436369896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,8191,0.010469333579142889
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,8191,0.008869333192706108
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,16383,0.013072000195582708
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,16383,0.011173332730929056
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,32767,0.01682666689157486
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,32767,0.01695466662446658
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,65535,0.025055999557177227
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,65535,0.023792001108328503
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,float16,8191,0.15401066342989603
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,1,0.007050666958093643
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,1,0.006735999758044879
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,3,0.006735999758044879
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,3,0.006671999891599019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,7,0.007002666592597961
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,7,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,15,0.006624000146985054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,15,0.006666666517655055
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,31,0.006634666894872983
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,31,0.006602666651209195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,63,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,63,0.006693333387374878
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,127,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,127,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,255,0.006677333265542984
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,255,0.006458666796485583
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,32767,0.30373867352803546
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,511,0.007029333462317784
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,15,0.006709333509206772
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,1023,0.008639999975760778
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,1023,0.008789333204428354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,2047,0.008821333448092142
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,2047,0.00855466661353906
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,4095,0.008586666857202848
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,4095,0.00860799973209699
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,8191,0.009685333197315535
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,8191,0.008682666967312494
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,16383,0.01302933320403099
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,16383,0.012831999609867731
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,32767,0.01714666684468587
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,2047,0.00855466661353906
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,32767,0.017231999586025875
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,65535,0.021231998999913532
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,131071,0.03898133337497711
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,131071,0.021205333371957142
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,1,0.006720000257094701
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,1,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,3,0.007104000076651573
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,float16,131071,0.029093332588672638
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,131071,0.0272533322374026
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,7,0.006677333265542984
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,7,0.006645333642760913
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,15,0.006640000268816948
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,31,0.006981333096822103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,31,0.006575999781489372
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,63,0.006805333619316419
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,63,0.006624000146985054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,127,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,127,0.007087999954819679
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,255,0.006666666517655055
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,511,0.0069973332186539965
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,1023,0.008762666955590248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,1023,0.008703999842206636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,2047,0.008943999807039896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,2047,0.008725333337982496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,4095,0.00892800030608972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,float16,511,0.00867733359336853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,4095,0.008725333337982496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,8191,0.0099093330403169
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,8191,0.00860799973209699
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,16383,0.0141546664138635
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,16383,0.013199999928474426
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,32767,0.017237332959969837
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,65535,0.03347733368476232
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,65535,0.016965333372354507
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,131071,0.033471999069054924
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,65535,0.018901333212852478
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,1,0.007791999727487564
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,3,0.006666666517655055
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,3,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,15,0.006560000280539195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,3,0.006597333277265231
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,7,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,7,0.006655999769767125
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,255,0.007184000064929326
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,15,0.006591999903321266
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,15,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,511,0.008592000231146812
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,31,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,31,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,63,0.006634666894872983
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,127,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,127,0.006666666517655055
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,255,0.006709333509206772
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,255,0.006954666847983996
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,511,0.008656000097592672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,511,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,1023,0.008762666955590248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,1023,0.008992000172535578
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,2047,0.008613333106040955
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,4095,0.008672000219424566
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,4095,0.00884799969693025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,8191,0.011973333855470022
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,8191,0.011050666371981302
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,16383,0.015157333264748255
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,16383,0.012858666479587555
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,float16,131071,0.05086933573087057
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,32767,0.02991466720898946
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,65535,0.029839999973773956
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,131071,0.08409066994984944
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,131071,0.04925866425037384
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,1,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,1,0.006511999915043513
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,3,0.006954666847983996
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,3,0.006400000303983688
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,7,0.00701333334048589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,7,0.007002666592597961
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,15,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,15,0.006949333474040031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,31,0.007189333438873291
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,31,0.006666666517655055
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,63,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,63,0.006704000135262807
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,2047,0.00884799969693025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,127,0.00702400008837382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,127,0.006757333253820737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,255,0.006741333131988843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,255,0.006602666651209195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,511,0.008922666932145754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,511,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,1023,0.008885333314538002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,32767,0.015216000378131866
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,1023,0.008799999952316284
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,2047,0.008912000184257826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,float16,65535,0.04780800143877665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,2047,0.008010666817426682
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,4095,0.00884799969693025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,8191,0.008746666833758354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,8191,0.010847999403874079
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,16383,0.013050666699806849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,16383,0.012847999731699625
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,32767,0.017184000462293625
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,32767,0.016986666868130367
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,65535,0.021274665991465252
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,65535,0.019029332945744198
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,131071,0.02123733361562093
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,1,0.00660800002515316
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,3,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,3,0.006629333520929019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,7,0.006640000268816948
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,63,0.006666666517655055
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,7,0.006517333288987477
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,15,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,31,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,15,0.0068800002336502075
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,31,0.006842666616042455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,63,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,127,0.006629333520929019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,127,0.006544000158707301
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,255,0.006629333520929019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,255,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,511,0.008762666955590248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,32767,0.015184000134468079
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,511,0.007157333195209503
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,1023,0.008789333204428354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,1023,0.008581333483258883
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,2047,0.008853333070874214
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,4095,0.009072000160813332
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,4095,0.008639999975760778
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,8191,0.01071999967098236
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,8191,0.008789333204428354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,1,0.006570666407545407
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,16383,0.014538666854302088
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,32767,0.017173333714405697
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,32767,0.015263999501864115
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,65535,0.0336053321758906
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,65535,0.017029333859682083
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,131071,0.03569599986076355
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,131071,0.033514666060606636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,1,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,3,0.006671999891599019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,3,0.006773333375652631
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,63,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,7,0.006778666749596596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,7,0.006677333265542984
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,15,0.006714666883150737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,15,0.006597333277265231
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,31,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,31,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,2047,0.008629333227872849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,63,0.006618666773041089
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,127,0.006725333631038666
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,127,0.006624000146985054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,255,0.0068800002336502075
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,float16,4095,0.008661333471536636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,255,0.006688000013430913
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,16383,0.012896000097195307
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,511,0.008602666358153025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,511,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,1023,0.008629333227872849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,2047,0.008661333471536636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,2047,0.00854399986565113
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,4095,0.0086666668454806
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,4095,0.008687999720374743
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,131071,0.05189866820971171
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,8191,0.011909333368142446
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,8191,0.010682666053374609
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,16383,0.014874666929244995
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,16383,0.013114667187134424
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,1,0.006629333520929019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,32767,0.015146666516860327
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,32767,0.02956799914439519
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,65535,0.04841066896915436
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,65535,0.031141333281993866
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,131071,0.04941866795221964
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,131071,0.08438400427500407
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,1,0.006720000257094701
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,3,0.00660800002515316
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,3,0.006591999903321266
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,7,0.00702400008837382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,15,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,15,0.006954666847983996
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,31,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,31,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,float16,63,0.00679466687142849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,63,0.006805333619316419
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,63,0.00702400008837382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,127,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,255,0.0069866664707660675
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,255,0.006800000245372455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,1023,0.0086666668454806
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,511,0.00871999996403853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,511,0.007050666958093643
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,1023,0.008896000062425932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,1023,0.008714666590094566
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,float16,1,0.007045333584149678
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,2047,0.00890666681031386
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,2047,0.00874133345981439
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,4095,0.01091733326514562
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,8191,0.014378666877746582
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,8191,0.012639999389648438
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,16383,0.029125332832336426
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,1,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,32767,0.047279998660087585
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,32767,0.02934933453798294
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,7,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,65535,0.08264533181985219
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,65535,0.04778666794300079
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,131071,0.15430933237075806
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,131071,0.08431466420491536
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,1,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,1,0.0069973332186539965
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,3,0.007327999919652939
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,3,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,7,0.007055999711155891
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,15,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,15,0.006831999868154526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,31,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,31,0.007007999966541926
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,63,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,63,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,127,0.0069440001000960665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,127,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,255,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,255,0.006821333120266597
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,511,0.008736000085870424
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,float16,4095,0.011152000476916632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,1023,0.009423999736706415
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,16383,0.014901333798964819
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,2047,0.011125333607196808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,2047,0.010725333044926325
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,4095,0.027066667874654133
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,4095,0.012879999975363413
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,8191,0.045642669002215065
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,8191,0.02701866626739502
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,1,0.008714666590094566
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,1,0.007002666592597961
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,16383,0.08269333342711131
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,16383,0.046069333950678505
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,7,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,127,0.006698666761318843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,3,0.008896000062425932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,3,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,7,0.008565333361426989
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,7,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,15,0.008485333373149237
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,31,0.008661333471536636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,15,0.007002666592597961
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,63,0.008623999853928884
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,63,0.006949333474040031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,127,0.008650666723648706
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,255,0.008687999720374743
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,255,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,511,0.008943999807039896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,511,0.008714666590094566
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,1023,0.010794666906197866
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,1023,0.00902399979531765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,1023,0.009018666421373686
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,2047,0.02629333237806956
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,2047,0.012602667013804117
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,4095,0.04450133442878723
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,8191,0.08142399787902832
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,8191,0.045514668027559914
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,1,0.00903466654320558
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,float16,16383,0.15491732954978943
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,1,0.008709333216150602
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,16383,0.082997332016627
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,3,0.008976000050703684
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,7,0.008693333094318708
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,3,0.008602666358153025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,7,0.00979200005531311
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,31,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,15,0.008938666433095932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,15,0.00854399986565113
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,31,0.008943999807039896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,31,0.00867733359336853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,127,0.00707733320693175
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,63,0.008954666554927826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,63,0.00878399983048439
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,float16,511,0.00878399983048439
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,127,0.008997333546479544
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,127,0.008912000184257826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,255,0.009072000160813332
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,255,0.00879466657837232
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,511,0.010970667004585266
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,511,0.01145600030819575
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,2047,0.026943999032179516
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,1023,0.013034666577974955
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,1023,0.026736001173655193
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,4095,0.08061333497365315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,4095,0.027066667874654133
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,8191,0.08270933230717976
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,4095,0.04610133171081543
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,8191,0.15309866269429526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,1,0.012144000579913458
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,3,0.012410666793584824
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,3,0.010965333630641302
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,7,0.010826667149861654
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,7,0.011877333124478659
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,16383,0.1581546664237976
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,16383,0.29897600412368774
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,15,0.012794667234023413
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,15,0.010949333508809408
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,31,0.012810666114091873
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,31,0.01101333275437355
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,127,0.010938666760921478
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,63,0.010890666395425797
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,255,0.012698666503032049
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,63,0.012746666868527731
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,float16,2047,0.04470400015513102
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,511,0.026714667677879333
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,511,0.01333333303531011
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,1023,0.027813332776228588
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,1023,0.046069333950678505
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,2047,0.08083733419577281
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,4095,0.08224000036716461
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,1,0.010650667051474253
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,2047,0.045706664522488914
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,1,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,8191,0.15439466635386148
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,1,0.008063999935984612
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,3,0.006773333375652631
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,3,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,7,0.006842666616042455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,7,0.006586666529377301
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,15,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,15,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,16383,0.6071360111236572
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,31,0.006714666883150737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,31,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,127,0.012805332740147909
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,63,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,127,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,63,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,127,0.0069919998447100324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,255,0.010661333799362183
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,511,0.008799999952316284
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,511,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,1023,0.009061333412925402
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,2047,0.008639999975760778
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,2047,0.008736000085870424
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,4095,0.008997333546479544
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,4095,0.008709333216150602
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,8191,0.010677333921194077
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,8191,0.00879466657837232
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,16383,0.012831999609867731
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,32767,0.017370666066805523
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,32767,0.015333333363135656
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,4095,0.15707733233769736
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,65535,0.03253333270549774
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,65535,0.017157333592573803
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,131071,0.049957334995269775
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,131071,0.03306666761636734
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,1,0.007034666836261749
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,1,0.006618666773041089
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,3,0.007045333584149678
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,3,0.006634666894872983
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,16383,0.3036853273709615
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,7,0.007055999711155891
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,7,0.006709333509206772
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,15,0.006773333375652631
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,31,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,255,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,255,0.006480000292261441
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,15,0.006709333509206772
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,31,0.006655999769767125
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,63,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,63,0.006741333131988843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,127,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,127,0.006629333520929019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,255,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,511,0.008639999975760778
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,511,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,1023,0.008581333483258883
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,1023,0.008703999842206636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,2047,0.008597333605090777
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,2047,0.008432000254591307
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,4095,0.008938666433095932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,4095,0.008896000062425932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,8191,0.012634667257467905
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,8191,0.01110400011142095
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,16383,0.015205333630243937
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,float16,16383,0.014138666292031607
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,16383,0.013114667187134424
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,32767,0.03092266619205475
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,65535,0.04894400139649709
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,131071,0.08501866459846497
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,131071,0.04987733562787374
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,float16,8191,0.30349334081013996
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,1,0.007029333462317784
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,3,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,3,0.007007999966541926
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,7,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,7,0.006741333131988843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,15,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,15,0.006575999781489372
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,31,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,31,0.00679466687142849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,63,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,63,0.006591999903321266
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,127,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,127,0.006773333375652631
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,255,0.007018666714429855
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,float16,255,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,255,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,511,0.008698666468262672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,511,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,1023,0.009056000038981438
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,1023,0.008714666590094566
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,2047,0.008767999708652496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,2047,0.009045333291093508
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,4095,0.010922666639089584
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,4095,0.010981333752473196
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,8191,0.013834666460752487
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,8191,0.012815999488035837
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,32767,0.01571200042963028
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,16383,0.029167999823888142
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,16383,0.01516266663869222
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,65535,0.030991998811562855
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,65535,0.08262399832407634
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,65535,0.048570667703946434
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,131071,0.08564266562461853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,131071,0.15565866231918335
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,1,0.006704000135262807
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,1,0.006735999758044879
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,3,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,3,0.006725333631038666
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,15,0.006762666627764702
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,15,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,31,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,31,0.006949333474040031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,63,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,63,0.006981333096822103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,1023,0.00860799973209699
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,127,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,127,0.00707733320693175
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,255,0.006629333520929019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,255,0.007130666946371396
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,511,0.00874133345981439
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,511,0.007018666714429855
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,1023,0.008736000085870424
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,1023,0.008992000172535578
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,float16,32767,0.046613335609436035
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,32767,0.02945599953333537
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,2047,0.008912000184257826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,2047,0.010751999914646149
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,4095,0.01301866645614306
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,4095,0.010805333654085795
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,8191,0.012885333349307379
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,8191,0.027082666754722595
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,16383,0.04472533365090688
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,16383,0.02734400083621343
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,1,0.006751999879876773
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,32767,0.08077333370844524
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,7,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,65535,0.153029332558314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,float16,131071,0.29846400022506714
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,1,0.008650666723648706
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,1,0.0069759997228781385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,3,0.008682666967312494
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,7,0.008623999853928884
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,3,0.006800000245372455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,7,0.00666133314371109
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,15,0.008767999708652496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,15,0.0069973332186539965
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,31,0.008586666857202848
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,31,0.006949333474040031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,63,0.008954666554927826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,63,0.007061333085099856
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,127,0.008816000074148178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,127,0.006981333096822103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,255,0.00867733359336853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,32767,0.0454720010360082
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,511,0.008896000062425932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,7,0.00679466687142849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,511,0.008672000219424566
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,1023,0.011002667248249054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,131071,0.15635200341542563
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,1023,0.008805333326260248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,2047,0.02701333413521449
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,2047,0.012869333227475485
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,4095,0.0458133320013682
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,float16,8191,0.08225066463152568
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,4095,0.027072000006834667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,3,0.008613333106040955
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,8191,0.04571733375390371
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,1,0.009093333035707474
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,1,0.008634666601816813
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,3,0.008650666723648706
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,7,0.00867733359336853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,7,0.008896000062425932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,15,0.00871999996403853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,255,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,15,0.00873066671192646
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,65535,0.08282666901747386
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,31,0.008938666433095932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,31,0.008863999818762144
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,63,0.008767999708652496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,127,0.008890666688481966
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,127,0.008922666932145754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,255,0.009018666421373686
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,255,0.008709333216150602
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,511,0.011359999577204386
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,511,0.010480000327030817
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,1023,0.026357332865397137
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,1023,0.013050666699806849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,4095,0.08265066643555959
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,4095,0.04585599899291992
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,2047,0.02741333345572154
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,8191,0.1569386621316274
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,1,0.010762666662534079
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,1,0.011381333072980246
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,3,0.012613333761692047
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,8191,0.08335466186205547
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,7,0.012341332932313284
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,3,0.010629333555698395
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,7,0.010911999891201654
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,15,0.011616000284751257
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,31,0.012928000340859095
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,31,0.010735999792814255
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,63,0.008976000050703684
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,63,0.012821332861979803
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,63,0.010784000158309937
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,127,0.01268799975514412
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,127,0.011002667248249054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,255,0.011994666109482447
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,255,0.010821333775917688
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,511,0.012896000097195307
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,511,0.02697066714366277
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,2047,0.0821919987599055
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,1023,0.04550399879614512
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,1023,0.02779199928045273
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,2047,0.04621866842110952
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,4095,0.08422933022181193
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,4095,0.1544373333454132
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,1,0.014991999914248785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,15,0.010805333654085795
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,1,0.01709866647919019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,3,0.014933332800865173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,3,0.016837333639462788
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,7,0.017136000096797943
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,7,0.014890667051076889
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,8191,0.1583039959271749
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,15,0.016885332763195038
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,15,0.01482133318980535
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,31,0.017184000462293625
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,31,0.014970666418472925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,63,0.01709866647919019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,127,0.017184000462293625
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,float16,2047,0.04549333453178406
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,127,0.014831999937693277
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,63,0.014767999450365702
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,255,0.02535466601451238
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,255,0.014815999815861383
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,1023,0.045279999574025474
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,511,0.027114666998386383
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,511,0.04374399781227112
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,2047,0.15271466970443726
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,1023,0.08032000064849854
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,2047,0.08167999982833862
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,4095,0.15659733613332114
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,1,0.00901333304742972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,4095,0.29613866408665973
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,1,0.008767999708652496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,3,0.008853333070874214
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,7,0.00890666681031386
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,3,0.008565333361426989
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,15,0.008778666456540426
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,7,0.00895999992887179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,8191,0.30551467339197796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,float16,8191,0.5824480056762695
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,15,0.008570666735370954
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,float16,8191,0.3036213318506877
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,31,0.00891733355820179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,31,0.008938666433095932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,63,0.008997333546479544
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,63,0.008997333546479544
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,255,0.008943999807039896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,255,0.008522666369875273
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,511,0.012453333785136541
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,127,0.009125333279371262
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,1023,0.02698666602373123
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,1023,0.013050666699806849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,1,0.012170666207869848
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,2047,0.027386667827765148
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,float16,2047,0.04560000201066335
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,1,0.010773333410422007
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,3,0.01062400018175443
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,3,0.0124746672809124
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,15,0.012709333250919977
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,7,0.012784000486135483
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,7,0.010693332801262537
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,15,0.010960000256697336
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,31,0.011685332904259363
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,31,0.010735999792814255
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,511,0.010677333921194077
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,63,0.011674666156371435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,63,0.01073066641887029
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,127,0.012634667257467905
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,127,0.011039999624093374
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,255,0.01156266654531161
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,255,0.010709332923094431
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,511,0.027285332481066387
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,511,0.01309866706530253
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,1023,0.04587199787298838
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,1023,0.02759466568628947
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,1,0.017136000096797943
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,1,0.01492799942692121
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,127,0.008746666833758354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,3,0.016810666769742966
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,7,0.014933332800865173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,2047,0.04664533336957296
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,15,0.016805333395799
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,float16,2047,0.08293866614500682
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,3,0.014842666685581207
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,7,0.017055999487638474
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,15,0.014970666418472925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,31,0.016864000509182613
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,31,0.014805333067973455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,63,0.014912000546852747
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,127,0.014848000059525171
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,255,0.015605332950750986
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,511,0.044490665197372437
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,511,0.0273333340883255
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,63,0.017237332959969837
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,2047,0.15254933635393778
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,127,0.01720000058412552
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,1023,0.08041066428025563
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,2047,0.08426133791605632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,1,0.023050665855407715
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,float16,255,0.0266239990790685
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,1,0.026975999275843304
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,3,0.02716800073782603
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,3,0.023311999936898548
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,7,0.02319466571013133
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,15,0.027285332481066387
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,31,0.027248000105222065
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,31,0.02314666658639908
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,63,0.027402666707833607
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,7,0.027514666318893433
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,63,0.0233599990606308
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,127,0.029152000943819683
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,1023,0.04537599782148997
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,255,0.044453332821528115
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,255,0.028117333849271137
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,1023,0.15197333693504333
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,15,0.023237332701683044
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,511,0.08070933322111766
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,1023,0.08125333487987518
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,float16,2047,0.2948213418324788
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,127,0.02314666658639908
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,2047,0.153221329053243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,1,0.012821332861979803
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,1,0.010693332801262537
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,3,0.01071999967098236
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,7,0.012901333471139273
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,15,0.012997332960367203
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,7,0.010837333897749582
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,3,0.012543999900420507
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,15,0.010778666784365972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,31,0.012645332763592402
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,511,0.04577599962552389
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,31,0.010949333508809408
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,63,0.012821332861979803
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,127,0.010522666076819101
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,63,0.010666667173306147
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,255,0.012655999511480331
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,255,0.010879999647537867
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,511,0.027269333600997925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,511,0.013370666652917862
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,1023,0.04593066871166229
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,1,0.017125333348910015
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,1,0.014896000425020853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,3,0.017125333348910015
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,1023,0.02870933214823405
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,float16,127,0.012757333616415659
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,7,0.016757333030303318
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,3,0.014981333166360855
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,15,0.01471466695268949
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,15,0.017130666722853977
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,31,0.016997333616018295
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,7,0.014959999670584997
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,31,0.01482133318980535
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,63,0.01685333376129468
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,63,0.015029333531856537
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,127,0.016778666526079178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,127,0.015018666783968607
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,255,0.014901333798964819
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,511,0.0454773356517156
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,511,0.027232001225153606
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,1,0.02733866622050603
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,1,0.023247999449570973
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,1023,0.08244266609350841
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,3,0.027263998985290527
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,1023,0.04656533400217692
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,3,0.02333866556485494
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,float16,255,0.02733866622050603
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,7,0.027114666998386383
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,7,0.022954667607943218
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,15,0.02734400083621343
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,31,0.027471999327341717
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,63,0.02733866622050603
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,63,0.022991999983787537
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,127,0.029834667841593426
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,127,0.023077333966890972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,255,0.045797333121299744
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,255,0.02916266769170761
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,31,0.023039999107519787
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,15,0.023285334308942158
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,511,0.08044800162315369
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,511,0.04604800045490265
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,3,0.04781333108743032
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,3,0.03963200002908707
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,1,0.039520000418027244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,1,0.04774933556715647
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,float16,1023,0.15285332997639975
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,15,0.04783466458320618
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,15,0.039605334401130676
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,31,0.03969600051641464
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,7,0.039450667798519135
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,7,0.04775466521581014
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,1023,0.08241066833337148
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,63,0.04032533367474874
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,127,0.049882665276527405
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,127,0.04099733382463455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,31,0.04854399959246317
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,1,0.007018666714429855
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,1,0.006773333375652631
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,511,0.15268799662590027
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,3,0.0069973332186539965
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,7,0.007082666580875714
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,63,0.04956266780694326
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,255,0.047594666481018066
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,255,0.08024533092975616
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,15,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,31,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,63,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,63,0.006666666517655055
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,511,0.08226133386294048
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,127,0.007018666714429855
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,1023,0.15411200126012167
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,float16,1023,0.2956693371136983
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,127,0.006730666384100914
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,511,0.008623999853928884
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,511,0.0069440001000960665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,1023,0.00874133345981439
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,1023,0.008874666566650072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,2047,0.008602666358153025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,2047,0.008922666932145754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,4095,0.00892800030608972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,31,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,4095,0.009039999917149544
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,8191,0.011690666278203329
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,8191,0.010944000134865442
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,16383,0.014933332800865173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,32767,0.02976000060637792
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,3,0.006634666894872983
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,32767,0.016842667013406754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,65535,0.048101335763931274
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,65535,0.03088533381621043
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,131071,0.08506666620572408
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,131071,0.04978133241335551
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,1,0.006741333131988843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,3,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,3,0.006773333375652631
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,7,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,7,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,15,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,15,0.006575999781489372
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,255,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,31,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,63,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,63,0.006704000135262807
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,127,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,127,0.006778666749596596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,255,0.006746666505932808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,15,0.006810666372378667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,255,0.006746666505932808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,511,0.008576000109314919
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,511,0.006810666372378667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,1023,0.00902399979531765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,1023,0.008805333326260248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,2047,0.00897066667675972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,2047,0.00877333308259646
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,4095,0.010949333508809408
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,4095,0.010773333410422007
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,8191,0.014111999422311783
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,1,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,8191,0.012896000097195307
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,16383,0.029178666571776073
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,float16,255,0.006671999891599019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,16383,0.014794666320085526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,32767,0.047770669062932335
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,32767,0.02924266705910365
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,65535,0.08592533071835835
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,1,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,131071,0.1572373310724894
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,float16,31,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,1,0.006602666651209195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,3,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,3,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,7,0.006709333509206772
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,7,0.006677333265542984
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,7,0.006693333387374878
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,15,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,15,0.006981333096822103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,31,0.0069919998447100324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,31,0.006570666407545407
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,63,0.006704000135262807
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,127,0.00666133314371109
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,127,0.007002666592597961
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,255,0.007007999966541926
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,255,0.006746666505932808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,511,0.0086666668454806
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,511,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,16383,0.012858666479587555
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,1023,0.009050666665037474
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,1023,0.008767999708652496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,2047,0.010714666297038397
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,2047,0.008613333106040955
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,4095,0.010725333044926325
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,8191,0.027066667874654133
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,8191,0.012805332740147909
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,16383,0.027263998985290527
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,32767,0.08070933322111766
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,32767,0.04677866895993551
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,65535,0.04795200129350027
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,65535,0.15265599886576334
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,65535,0.08355733752250671
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,131071,0.08830933769543965
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,1,0.007018666714429855
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,1,0.006831999868154526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,131071,0.2990773320198059
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,131071,0.15775466958681741
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,3,0.0069973332186539965
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,63,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,3,0.006831999868154526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,7,0.0069759997228781385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,15,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,15,0.0069919998447100324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,31,0.00679466687142849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,31,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,63,0.0069440001000960665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,63,0.007007999966541926
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,127,0.006981333096822103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,127,0.006805333619316419
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,255,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,255,0.007141333073377609
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,511,0.007631999750932057
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,511,0.008736000085870424
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,1023,0.009946666657924652
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,4095,0.012709333250919977
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,1023,0.0086666668454806
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,2047,0.011226666470368704
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,2047,0.01071999967098236
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,4095,0.012991999586423239
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,float16,16383,0.04544533292452494
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,8191,0.027306665976842243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,8191,0.04420266548792521
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,16383,0.0447626660267512
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,16383,0.0806826651096344
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,32767,0.08224000036716461
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,32767,0.15202132860819498
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,65535,0.29627732435862225
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,131071,0.29703466097513836
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,131071,0.5857333342234293
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,7,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,float16,4095,0.026943999032179516
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,1,0.015018666783968607
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,1,0.016832000265518825
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,65535,0.15382933616638184
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,3,0.016821333517630894
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,7,0.016805333395799
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,3,0.015178666760524115
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,7,0.014778666198253632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,15,0.01716800034046173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,15,0.014783999572197596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,31,0.01714666684468587
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,31,0.014975999792416891
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,63,0.016890666137139004
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,63,0.01525866612792015
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,127,0.014864000181357065
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,127,0.01716800034046173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,255,0.0272533322374026
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,255,0.014831999937693277
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,511,0.027664000789324444
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,float16,511,0.04563733438650767
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,1,0.02739199995994568
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,1,0.0229120006163915
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,3,0.02754133443037669
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,3,0.023413332800070446
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,7,0.026933332284291584
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,7,0.02327466756105423
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,31,0.027093333502610523
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,15,0.027434666951497395
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,31,0.02292266736427943
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,15,0.023039999107519787
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,63,0.02775466690460841
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,63,0.023082666099071503
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,127,0.029301332930723827
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,127,0.02288000037272771
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,255,0.028991999725500744
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,511,0.08262399832407634
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,1,0.04819199939568838
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,511,0.0469760000705719
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,1,0.03935466706752777
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,3,0.04784533381462097
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,float16,255,0.04574933151404063
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,3,0.03965866565704346
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,7,0.048298666874567665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,15,0.04781333108743032
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,31,0.048112000028292336
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,7,0.03973866750796636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,31,0.03958400090535482
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,15,0.03978666663169861
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,63,0.04888000090916952
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,63,0.039647998909155525
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,127,0.05020800232887268
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,127,0.040336000422636666
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,255,0.08183999856313069
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,255,0.048197334011395775
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,1,0.088128000497818
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,1,0.07236266632874806
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,float16,511,0.15386133392651877
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,511,0.0842186709245046
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,3,0.08741866548856099
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,3,0.07237333556016286
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,7,0.08874133229255676
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,15,0.08802666266759236
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,7,0.07239466905593872
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,63,0.08855467041333516
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,63,0.07221333185831706
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,127,0.09087999661763509
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,127,0.0727946658929189
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,31,0.08902399738629659
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,15,0.072202667593956
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,31,0.07267733414967854
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,255,0.08907199899355571
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,255,0.15333333611488342
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,1,0.02333866556485494
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,1,0.027114666998386383
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,3,0.027610667049884796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,511,0.15414933363596597
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,float16,511,0.29627732435862225
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,3,0.02309333284695943
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,7,0.02752000093460083
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,7,0.02319466571013133
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,15,0.027056001126766205
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,15,0.023024000227451324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,31,0.027535999814669292
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,63,0.022954667607943218
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,127,0.03125333289305369
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,127,0.023050665855407715
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,31,0.023018665611743927
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,63,0.027376001079877216
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,float16,255,0.04601066807905833
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,1,0.04854399959246317
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,255,0.029301332930723827
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,1,0.03937600056330363
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,3,0.049327999353408813
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,7,0.04808533191680908
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,3,0.03937600056330363
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,7,0.039605334401130676
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,15,0.04990933338801066
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,15,0.039477333426475525
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,63,0.04780266682306925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,63,0.03929600119590759
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,127,0.05212266743183136
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,31,0.048997332652409874
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,31,0.03972266614437103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,float16,255,0.0839359958966573
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,127,0.041749333341916404
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,255,0.04984533290068308
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,1,0.08860266208648682
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,1,0.07252799967924754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,3,0.07213866710662842
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,3,0.08896533648173015
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,7,0.08904000123341878
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,7,0.07334933181603749
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,15,0.0888426701227824
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,15,0.0724480003118515
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,31,0.08802133798599243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,31,0.07271466652552287
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,63,0.08872532844543457
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,63,0.07273066540559132
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,127,0.09269866347312927
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,127,0.07262399792671204
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,1,0.1674826741218567
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,float16,255,0.15620799859364828
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,1,0.13573867082595825
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,255,0.08922132849693298
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,3,0.16662399967511496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,3,0.1366986632347107
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,15,0.16759467124938965
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,31,0.16801599661509195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,7,0.13645866513252258
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,7,0.16814400752385458
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,63,0.16767466068267822
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,15,0.1365226705869039
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,31,0.135861337184906
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,1,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,3,0.006709333509206772
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,1,0.00706666645904382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,63,0.1376586655775706
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,7,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,7,0.006773333375652631
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,15,0.0069919998447100324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,15,0.00679466687142849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,31,0.006640000268816948
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,63,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,3,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,63,0.006688000013430913
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,127,0.007130666946371396
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,127,0.006671999891599019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,255,0.006778666749596596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,255,0.006714666883150737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,511,0.008703999842206636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,511,0.007002666592597961
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,1023,0.00867733359336853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,1023,0.008634666601816813
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,2047,0.009008000294367472
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,2047,0.008645333349704742
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,4095,0.011039999624093374
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,4095,0.010794666906197866
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,8191,0.013568000247081121
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,8191,0.012901333471139273
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,16383,0.014959999670584997
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,16383,0.029018667836983997
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,32767,0.047466665506362915
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,32767,0.029258665939172108
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,127,0.17266666889190674
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,31,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,1,0.006789333497484525
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,65535,0.048570667703946434
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,1,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,3,0.006741333131988843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,7,0.006618666773041089
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,7,0.006773333375652631
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,15,0.006831999868154526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,15,0.006821333120266597
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,31,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,63,0.006768000001708667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,63,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,127,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,127,0.006805333619316419
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,255,0.006720000257094701
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,255,0.006757333253820737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,511,0.008650666723648706
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,511,0.007237333183487256
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,1023,0.00895999992887179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,3,0.006586666529377301
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,1023,0.009733333562811216
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,2047,0.010650667051474253
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,2047,0.008863999818762144
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,4095,0.013077333569526672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,4095,0.010634666929642359
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,8191,0.013093333691358566
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,16383,0.045519997676213585
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,float16,65535,0.0853653351465861
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,16383,0.027424000203609467
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,127,0.13758933544158936
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,32767,0.08253333469231923
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,32767,0.04612799982229868
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,65535,0.15676800409952799
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,1,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,65535,0.08637332916259766
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,31,0.006682666639486949
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,3,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,3,0.007071999832987785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,7,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,7,0.006949333474040031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,15,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,15,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,31,0.006842666616042455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,31,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,63,0.006602666651209195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,63,0.00725333330531915
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,127,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,127,0.007045333584149678
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,255,0.006954666847983996
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,255,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,1,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,511,0.008623999853928884
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,511,0.008053333188096682
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,1023,0.009530666594703993
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,1023,0.008821333448092142
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,2047,0.011173332730929056
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,2047,0.010832000523805618
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,4095,0.0269813338915507
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,float16,8191,0.02700799951950709
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,8191,0.04468800127506256
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,16383,0.04577066500981649
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,16383,0.08059733112653096
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,32767,0.1523146629333496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,255,0.16556800405184427
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,32767,0.08328000207742055
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,float16,255,0.29847466945648193
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,1,0.007989333321650824
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,float16,65535,0.29815999666849774
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,65535,0.15596800049146017
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,3,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,7,0.007055999711155891
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,7,0.009088000282645226
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,15,0.0069440001000960665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,31,0.00850133349498113
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,31,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,4095,0.012752000242471695
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,63,0.006762666627764702
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,8191,0.027386667827765148
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,127,0.007178666690985362
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,127,0.00867733359336853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,255,0.00860799973209699
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,255,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,511,0.00892800030608972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,511,0.008656000097592672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,1023,0.010938666760921478
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,1023,0.008890666688481966
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,2047,0.025407999753952026
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,2047,0.012639999389648438
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,4095,0.043552001317342125
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,15,0.008634666601816813
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,8191,0.08030400176843007
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,8191,0.04433600107828776
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,16383,0.1520746648311615
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,16383,0.08050133287906647
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,63,0.008896000062425932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,32767,0.295909325281779
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,32767,0.1530186633268992
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,1,0.0069919998447100324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,1,0.006624000146985054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,65535,0.29709867636362713
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,3,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,7,0.0069759997228781385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,3,0.007055999711155891
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,65535,0.5857439835866293
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,15,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,7,0.006800000245372455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,15,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,31,0.007135999699433644
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,63,0.00679466687142849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,63,0.006597333277265231
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,127,0.006768000001708667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,127,0.006677333265542984
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,255,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,511,0.008581333483258883
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,4095,0.025946666797002155
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,31,0.006720000257094701
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,1023,0.00879466657837232
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,511,0.007391999786098798
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,1023,0.00867733359336853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,2047,0.010693332801262537
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,1,0.006549333532651265
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,2047,0.00902399979531765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,8191,0.02701333413521449
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,4095,0.012736000120639801
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,4095,0.011066666493813196
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,8191,0.013023999830087027
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,float16,3,0.009018666421373686
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,32767,0.082997332016627
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,16383,0.02754666656255722
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,32767,0.04655466477076212
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,1,0.00726400005320708
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,3,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,7,0.007167999943097432
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,7,0.00707733320693175
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,15,0.007055999711155891
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,15,0.011296000331640244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,255,0.006805333619316419
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,31,0.007114666824539502
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,31,0.007658666620651881
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,63,0.006789333497484525
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,65535,0.15865600109100342
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,127,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,127,0.006671999891599019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,float16,16383,0.04540266593297323
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,255,0.0069973332186539965
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,511,0.008821333448092142
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,511,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,1,0.006981333096822103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,1023,0.00972800018886725
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,1023,0.008938666433095932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,2047,0.011141333729028702
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,3,0.007418666655818622
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,2047,0.010960000256697336
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,4095,0.027242665489514668
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,4095,0.013189333180586496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,8191,0.04553600152333578
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,8191,0.027295999228954315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,16383,0.08231999973456065
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,16383,0.046575998266537987
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,32767,0.15677866339683533
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,32767,0.08337600032488506
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,1,0.008757333581646284
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,1,0.006751999879876773
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,65535,0.08436266581217448
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,65535,0.30341867605845135
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,63,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,3,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,float16,255,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,7,0.008672000219424566
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,15,0.007061333085099856
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,7,0.0069919998447100324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,31,0.008757333581646284
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,31,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,63,0.008682666967312494
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,63,0.007162666569153468
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,127,0.008576000109314919
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,127,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,255,0.008661333471536636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,255,0.007184000064929326
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,511,0.008949333180983862
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,511,0.00860799973209699
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,1023,0.01099733387430509
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,1023,0.008965333302815756
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,2047,0.025850666066010792
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,2047,0.012650666137536367
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,4095,0.04407466451327006
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,4095,0.027130665878454845
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,8191,0.04548799991607666
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,8191,0.08062399923801422
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,16383,0.15292267004648843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,65535,0.158053328593572
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,1,0.007029333462317784
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,32767,0.29715200265248615
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,3,0.008992000172535578
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,32767,0.15627200404802957
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,1,0.006789333497484525
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,3,0.006778666749596596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,15,0.008709333216150602
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,7,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,float16,65535,0.5839893420537313
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,65535,0.3014613389968872
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,31,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,63,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,63,0.006720000257094701
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,31,0.006528000036875407
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,127,0.006629333520929019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,255,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,255,0.006618666773041089
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,511,0.00867733359336853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,511,0.006741333131988843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,1023,0.008736000085870424
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,1023,0.0069440001000960665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,2047,0.008698666468262672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,2047,0.008613333106040955
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,4095,0.008645333349704742
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,4095,0.008623999853928884
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,8191,0.008949333180983862
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,8191,0.009029333169261614
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,16383,0.012784000486135483
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,16383,0.08146133522192638
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,16383,0.01089599976936976
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,32767,0.016885332763195038
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,32767,0.015018666783968607
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,65535,0.025040000677108765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,65535,0.023290666441122692
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,131071,0.029509333272775013
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,131071,0.0271519993742307
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,1,0.006714666883150737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,1,0.006549333532651265
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,3,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,3,0.006597333277265231
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,7,0.007018666714429855
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,7,0.00702400008837382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,15,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,15,0.006538666784763336
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,15,0.006575999781489372
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,31,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,31,0.006735999758044879
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,63,0.006800000245372455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,63,0.00660800002515316
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,127,0.0069866664707660675
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,127,0.00666133314371109
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,255,0.006842666616042455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,255,0.006735999758044879
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,511,0.00867733359336853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,127,0.006730666384100914
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,511,0.0068800002336502075
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,1023,0.008752000207702318
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,1023,0.008565333361426989
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,2047,0.008752000207702318
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,2047,0.00884799969693025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,4095,0.008703999842206636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,4095,0.008656000097592672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,8191,0.008837333569924036
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,8191,0.008752000207702318
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,16383,0.013002666334311167
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,16383,0.012768000364303589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,32767,0.017125333348910015
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,32767,0.017114666601022083
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,65535,0.021104000508785248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,65535,0.019402666638294857
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,131071,0.03939199944337209
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,131071,0.021253332495689392
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,1,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,3,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,3,0.006565333033601443
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,7,0.0069919998447100324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,7,0.006640000268816948
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,15,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,15,0.006725333631038666
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,31,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,31,0.006789333497484525
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,63,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,63,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,127,0.007141333073377609
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,127,0.006768000001708667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,255,0.006682666639486949
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,255,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,511,0.008965333302815756
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,511,0.007370666911204656
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,1023,0.008757333581646284
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,float16,3,0.006655999769767125
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,1023,0.007797333101431529
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,7,0.006565333033601443
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,float16,15,0.006624000146985054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,4095,0.00895999992887179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,4095,0.008778666456540426
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,8191,0.009445333232482275
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,8191,0.008949333180983862
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,16383,0.014709333578745524
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,16383,0.012762666990359625
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,32767,0.015157333264748255
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,65535,0.033402666449546814
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,131071,0.05090666810671488
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,131071,0.033471999069054924
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,1,0.006714666883150737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,1,0.006544000158707301
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,3,0.006560000280539195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,3,0.006746666505932808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,7,0.006800000245372455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,1,0.006773333375652631
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,7,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,15,0.006602666651209195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,15,0.006522666662931442
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,31,0.006597333277265231
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,63,0.006650666395823161
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,63,0.006586666529377301
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,127,0.007002666592597961
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,127,0.00655466690659523
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,255,0.006725333631038666
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,255,0.006602666651209195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,511,0.008485333373149237
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,511,0.007045333584149678
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,1023,0.008645333349704742
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,1023,0.008277333031098047
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,2047,0.008789333204428354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,4095,0.008725333337982496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,2047,0.00874133345981439
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,4095,0.00884799969693025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,8191,0.009237333511312803
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,2047,0.008869333192706108
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,16383,0.012698666503032049
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,8191,0.008709333216150602
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,16383,0.012810666114091873
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,32767,0.01717866708834966
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,32767,0.01729600007335345
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,65535,0.021013334393501282
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,131071,0.03899733225504557
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,65535,0.017077332983414333
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,131071,0.021146667500336964
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,1,0.0069866664707660675
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,1,0.006618666773041089
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,3,0.006714666883150737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,7,0.006981333096822103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,7,0.006602666651209195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,15,0.006800000245372455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,15,0.006704000135262807
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,31,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,31,0.006746666505932808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,63,0.006735999758044879
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,63,0.006789333497484525
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,127,0.006640000268816948
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,255,0.006709333509206772
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,31,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,511,0.00874133345981439
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,511,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,1023,0.00898133342464765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,1023,0.008762666955590248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,2047,0.008901333436369896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,2047,0.00871999996403853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,float16,2047,0.008997333546479544
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,4095,0.008943999807039896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,8191,0.010351999973257383
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,65535,0.021157334248224895
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,8191,0.008938666433095932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,16383,0.014511999984582266
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,16383,0.013002666334311167
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,3,0.006570666407545407
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,32767,0.015322666615247726
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,65535,0.03313066562016805
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,65535,0.01704000060757001
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,131071,0.05118933320045471
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,131071,0.03453866640726725
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,127,0.006805333619316419
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,1,0.006666666517655055
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,3,0.0069919998447100324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,3,0.006634666894872983
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,7,0.007087999954819679
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,7,0.006688000013430913
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,15,0.006650666395823161
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,31,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,31,0.006650666395823161
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,4095,0.008805333326260248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,63,0.006842666616042455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,63,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,127,0.006586666529377301
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,127,0.006624000146985054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,float16,32767,0.017312000195185345
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,255,0.006640000268816948
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,255,0.006735999758044879
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,511,0.008821333448092142
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,511,0.007087999954819679
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,1023,0.008954666554927826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,float16,32767,0.016778666526079178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,1023,0.007690666864315669
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,2047,0.008623999853928884
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,2047,0.008623999853928884
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,4095,0.00874133345981439
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,4095,0.008922666932145754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,8191,0.011658667276302973
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,8191,0.01091733326514562
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,16383,0.014783999572197596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,16383,0.012965332716703415
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,32767,0.015610666324694952
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,32767,0.030597334106763203
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,65535,0.04816000163555145
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,65535,0.03107200066248576
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,255,0.007055999711155891
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,131071,0.08444266517957051
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,131071,0.048911998669306435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,1,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,1,0.006629333520929019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,3,0.006655999769767125
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,7,0.0069440001000960665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,15,0.006981333096822103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,float16,15,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,15,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,31,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,31,0.006629333520929019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,63,0.0068800002336502075
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,63,0.006570666407545407
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,127,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,255,0.006666666517655055
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,255,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,511,0.008698666468262672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,511,0.007589333380262057
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,1023,0.009349333122372627
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,1023,0.008896000062425932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,2047,0.010816000401973724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,2047,0.010682666053374609
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,4095,0.026954665780067444
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,4095,0.01313599944114685
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,8191,0.027658666173617046
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,1,0.006586666529377301
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,16383,0.082805335521698
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,16383,0.04731200138727824
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,3,0.007061333085099856
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,32767,0.15758933623631796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,1,0.009002666920423508
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,1,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,32767,0.08538666367530823
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,3,0.00891733355820179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,3,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,7,0.006842666616042455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,15,0.00867733359336853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,15,0.006735999758044879
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,31,0.008570666735370954
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,31,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,63,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,127,0.00855466661353906
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,127,0.006805333619316419
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,255,0.008613333106040955
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,255,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,511,0.008874666566650072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,8191,0.04563733438650767
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,1023,0.010901333143313726
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,1023,0.008965333302815756
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,2047,0.026389333109060924
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,2047,0.012639999389648438
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,7,0.00660800002515316
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,4095,0.044666667779286705
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,7,0.008656000097592672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,8191,0.08231466511885326
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,float16,127,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,8191,0.04553600152333578
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,16383,0.15439466635386148
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,63,0.009061333412925402
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,16383,0.08282666901747386
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,1,0.00933333362142245
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,1,0.008709333216150602
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,3,0.010496000448862711
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,float16,32767,0.30244266986846924
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,511,0.008672000219424566
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,3,0.008613333106040955
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,32767,0.15586666266123453
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,7,0.00921066664159298
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,7,0.008863999818762144
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,31,0.00897066667675972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,15,0.00903466654320558
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,31,0.00892800030608972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,127,0.008954666554927826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,127,0.00850133349498113
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,255,0.008816000074148178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,4095,0.0269813338915507
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,255,0.008682666967312494
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,511,0.011285333583752314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,511,0.010762666662534079
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,2047,0.044026667873064675
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,1023,0.012901333471139273
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,1023,0.02565866708755493
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,2047,0.027109332382678986
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,4095,0.08053333560625713
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,4095,0.045738667249679565
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,8191,0.1525973379611969
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,8191,0.08269866804281871
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,16383,0.2979733347892761
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,1,0.006768000001708667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,16383,0.15763733784357706
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,15,0.00879466657837232
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,3,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,63,0.008976000050703684
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,1,0.006693333387374878
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,7,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,3,0.006810666372378667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,15,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,15,0.006597333277265231
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,31,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,31,0.006709333509206772
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,63,0.006810666372378667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,63,0.006597333277265231
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,127,0.006618666773041089
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,127,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,255,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,255,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,511,0.00854399986565113
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,511,0.006805333619316419
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,1023,0.008618666479984919
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,1023,0.008714666590094566
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,2047,0.008565333361426989
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,2047,0.00877333308259646
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,4095,0.009050666665037474
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,4095,0.008709333216150602
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,8191,0.009973333527644476
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,8191,0.008656000097592672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,16383,0.014474666366974512
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,16383,0.01302933320403099
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,32767,0.015061333775520325
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,65535,0.033615998923778534
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,65535,0.017136000096797943
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,131071,0.051962668697039284
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,131071,0.03519999980926514
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,1,0.007018666714429855
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,1,0.006735999758044879
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,3,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,3,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,7,0.006709333509206772
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,7,0.006671999891599019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,15,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,15,0.006757333253820737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,31,0.006671999891599019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,31,0.006831999868154526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,63,0.006650666395823161
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,63,0.006570666407545407
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,127,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,127,0.006704000135262807
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,255,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,255,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,511,0.008736000085870424
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,511,0.006949333474040031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,1023,0.00914666677514712
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,63,0.008816000074148178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,7,0.007413333281874657
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,1023,0.008661333471536636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,2047,0.008656000097592672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,2047,0.008463999877373377
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,4095,0.009189333145817121
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,8191,0.010778666784365972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,16383,0.015077333897352219
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,8191,0.012650666137536367
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,16383,0.012949333836634954
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,32767,0.029696000119050343
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,float16,32767,0.5919093290964762
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,32767,0.30771734317143756
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,32767,0.01648533344268799
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,65535,0.04934933284918467
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,65535,0.031066666046778362
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,1,0.0064853330453236895
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,3,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,float16,32767,0.017103999853134155
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,1,0.007237333183487256
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,3,0.006581333155433337
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,7,0.006768000001708667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,7,0.006768000001708667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,float16,131071,0.08551999926567078
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,15,0.006842666616042455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,131071,0.04984533290068308
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,31,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,15,0.007141333073377609
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,63,0.00666133314371109
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,127,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,127,0.006693333387374878
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,255,0.006682666639486949
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,255,0.012597333639860153
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,511,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,511,0.00891733355820179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,2047,0.008672000219424566
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,2047,0.00878399983048439
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,1023,0.009125333279371262
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,4095,0.010928000013033548
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,8191,0.014293332894643148
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,8191,0.012730666746695837
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,4095,0.008954666554927826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,16383,0.029535998900731403
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,32767,0.04715733230113983
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,32767,0.02961066613594691
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,65535,0.08264533181985219
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,65535,0.04863466819127401
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,131071,0.1544266641139984
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,131071,0.08561600248018901
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,63,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,1,0.008943999807039896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,1023,0.00873066671192646
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,3,0.00874133345981439
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,1,0.00721066693464915
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,3,0.0069919998447100324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,float16,4095,0.010981333752473196
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,7,0.008576000109314919
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,7,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,16383,0.014906667172908783
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,31,0.008576000109314919
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,31,0.006954666847983996
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,63,0.00854399986565113
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,127,0.008634666601816813
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,127,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,255,0.008602666358153025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,255,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,511,0.00892800030608972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,31,0.006602666651209195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,1023,0.010805333654085795
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,1023,0.008645333349704742
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,2047,0.026389333109060924
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,2047,0.012794667234023413
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,4095,0.04542933404445648
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,4095,0.027093333502610523
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,15,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,15,0.009072000160813332
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,8191,0.045567999283472695
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,63,0.007040000210205714
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,1,0.008879999940594038
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,16383,0.15583466490109762
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,3,0.008997333546479544
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,1,0.008863999818762144
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,16383,0.08368000388145447
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,511,0.008586666857202848
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,7,0.00892800030608972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,7,0.008613333106040955
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,3,0.009279999881982803
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,15,0.008789333204428354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,31,0.00897066667675972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,31,0.008693333094318708
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,63,0.00891733355820179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,63,0.008592000231146812
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,127,0.008570666735370954
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,127,0.008933333059151968
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,255,0.008645333349704742
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,255,0.00897066667675972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,511,0.01081066702802976
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,float16,8191,0.08282133440176646
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,1023,0.013061333447694778
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,4095,0.08298133313655853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,2047,0.04549333453178406
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,2047,0.02741333345572154
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,8191,0.0844586690266927
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,8191,0.1550879975159963
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,15,0.009178666397929192
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,1,0.012645332763592402
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,1,0.011418666690587997
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,16383,0.3041226665178935
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,511,0.012138667205969492
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,3,0.012554666648308435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,float16,1023,0.026933332284291584
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,3,0.011434666812419891
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,7,0.012746666868527731
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,7,0.010992000500361124
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,4095,0.04646400113900503
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,15,0.010832000523805618
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,15,0.012650666137536367
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,31,0.010762666662534079
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,63,0.012757333616415659
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,31,0.013162666310866674
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,63,0.010944000134865442
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,127,0.012709333250919977
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,255,0.011328000575304031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,127,0.011258666714032492
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,255,0.010821333775917688
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,511,0.012997332960367203
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,511,0.02703999976317088
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,16383,0.1600266695022583
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,2047,0.08113066852092743
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,2047,0.045610666275024414
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,4095,0.15430933237075806
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,8191,0.30589866638183594
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,4095,0.08333333333333333
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,1,0.009039999917149544
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,3,0.00898133342464765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,1,0.008656000097592672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,16383,0.3059306740760803
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,16383,0.6063413222630819
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,1023,0.027269333600997925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,3,0.008682666967312494
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,7,0.009408000235756239
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,7,0.008559999987483025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,31,0.008656000097592672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,8191,0.15652799606323242
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,63,0.009098666409651438
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,15,0.009296000003814697
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,63,0.008639999975760778
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,31,0.008912000184257826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,15,0.008901333436369896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,127,0.00895999992887179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,255,0.008965333302815756
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,511,0.012896000097195307
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,511,0.010666667173306147
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,255,0.008858666444818178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,1023,0.027263998985290527
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,1023,0.012917333592971167
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,127,0.009685333197315535
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,2047,0.045791998505592346
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,2047,0.027647999425729115
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,1,0.010650667051474253
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,1,0.012858666479587555
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,3,0.013232000172138214
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,3,0.010725333044926325
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,7,0.012821332861979803
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,7,0.010661333799362183
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,float16,1023,0.04470400015513102
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,15,0.01293333371480306
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,4095,0.04819199939568838
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,31,0.012442667037248611
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,15,0.011424000064531961
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,63,0.012698666503032049
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,127,0.012202666451533636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,63,0.010709332923094431
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,255,0.01179733375708262
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,float16,4095,0.08230933547019958
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,511,0.027034667630990345
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,511,0.013072000195582708
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,2047,0.08261866867542267
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,127,0.010944000134865442
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,1023,0.02828266719977061
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,31,0.010890666395425797
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,2047,0.046394666035970054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,1,0.01687466725707054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,4095,0.1562986671924591
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,3,0.01687466725707054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,1,0.014762666076421738
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,4095,0.0839466651280721
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,7,0.016858667135238647
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,255,0.011231999844312668
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,7,0.014991999914248785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,15,0.017125333348910015
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,15,0.014709333578745524
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,3,0.014842666685581207
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,float16,1023,0.04577066500981649
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,31,0.01682666689157486
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,63,0.015072000523408255
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,127,0.017018667111794155
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,31,0.015295999745527903
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,127,0.014896000425020853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,255,0.026357332865397137
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,511,0.043680002291997276
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,63,0.017279999951521557
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,255,0.014981333166360855
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,511,0.02714666724205017
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,2047,0.1529706617196401
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,2047,0.082997332016627
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,4095,0.15923200050989786
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,4095,0.29628799359003705
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,float16,1023,0.0803306649128596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,1,0.012879999975363413
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,1023,0.04574400186538696
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,3,0.012645332763592402
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,1,0.011066666493813196
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,3,0.01089599976936976
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,7,0.012800000607967377
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,15,0.012746666868527731
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,7,0.010751999914646149
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,31,0.012938667088747025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,15,0.011066666493813196
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,31,0.01098666712641716
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,63,0.012831999609867731
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,127,0.012890666723251343
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,127,0.010890666395425797
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,255,0.012432000289360682
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,255,0.010703999549150467
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,511,0.027082666754722595
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,511,0.014085333794355392
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,1023,0.04553066690762838
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,1023,0.027776000400384266
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,63,0.010709332923094431
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,float16,2047,0.08335999647776286
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,1,0.01684800038735072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,1,0.014741333822409311
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,2047,0.04574933151404063
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,3,0.017152000218629837
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,3,0.015008000036080679
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,7,0.014853333433469137
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,15,0.014842666685581207
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,31,0.01709866647919019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,63,0.017263999829689663
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,15,0.017269333203633625
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,63,0.014933332800865173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,127,0.016800000021855038
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,31,0.014698666830857595
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,127,0.014917333920796713
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,255,0.02699733277161916
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,7,0.017194667210181553
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,255,0.015034666905800501
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,511,0.027141332626342773
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,511,0.045226668318112694
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,1,0.027263998985290527
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,1023,0.08150933186213176
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,1,0.02317333221435547
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,1023,0.04606399933497111
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,3,0.02717866748571396
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,7,0.02736533433198929
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,3,0.023168000082174938
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,2047,0.0846666693687439
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,float16,2047,0.15562132994333902
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,15,0.027690666417280834
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,31,0.023210667073726654
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,7,0.022944000860055287
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,15,0.023056000471115112
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,63,0.027295999228954315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,127,0.023189333577950794
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,255,0.04451733330885569
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,255,0.028730665644009907
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,31,0.027322667340437572
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,511,0.0805866668621699
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,511,0.04557866851488749
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,2047,0.15570132931073508
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,3,0.006954666847983996
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,3,0.006981333096822103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,7,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,127,0.029152000943819683
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,7,0.006757333253820737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,63,0.02310933421055476
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,15,0.006778666749596596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,31,0.007002666592597961
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,63,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,31,0.0069759997228781385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,15,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,63,0.006784000123540561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,1,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,127,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,127,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,255,0.007018666714429855
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,255,0.006954666847983996
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,511,0.008634666601816813
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,511,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,1023,0.008736000085870424
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,1023,0.00884799969693025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,2047,0.00892800030608972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,2047,0.008559999987483025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,4095,0.009018666421373686
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,8191,0.012853333105643591
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,8191,0.01097600037852923
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,16383,0.014970666418472925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,32767,0.031018666923046112
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,32767,0.016783999900023144
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,65535,0.049466664592425026
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,65535,0.031109333038330078
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,131071,0.08534933129946391
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,131071,0.04967466493447622
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,1,0.0069440001000960665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,1,0.0069759997228781385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,2047,0.2968906760215759
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,3,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,3,0.006789333497484525
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,7,0.006954666847983996
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,7,0.006528000036875407
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,15,0.006575999781489372
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,float16,1,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,31,0.006709333509206772
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,1023,0.08345066507657369
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,31,0.006704000135262807
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,63,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,float16,1023,0.15268266201019287
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,63,0.006624000146985054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,127,0.00655466690659523
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,255,0.006528000036875407
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,4095,0.008976000050703684
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,511,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,1023,0.008992000172535578
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,1023,0.008757333581646284
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,2047,0.009103999783595404
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,2047,0.00879466657837232
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,4095,0.011002667248249054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,4095,0.010853332777818045
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,8191,0.01421333352724711
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,8191,0.012757333616415659
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,16383,0.02958400050799052
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,16383,0.014975999792416891
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,32767,0.029509333272775013
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,32767,0.04764799773693085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,15,0.00795199970404307
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,65535,0.08442133665084839
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,65535,0.04774933556715647
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,131071,0.1581493318080902
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,1,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,131071,0.08618666728337605
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,1,0.006629333520929019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,3,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,3,0.00679466687142849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,511,0.008736000085870424
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,7,0.00706666645904382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,16383,0.013178666432698568
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,15,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,15,0.006704000135262807
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,31,0.006677333265542984
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,31,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,63,0.006655999769767125
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,127,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,127,0.00666133314371109
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,255,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,255,0.006693333387374878
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,1023,0.008629333227872849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,1023,0.008810666700204214
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,2047,0.010661333799362183
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,2047,0.008858666444818178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,4095,0.012671999633312225
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,4095,0.0106133334338665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,8191,0.027189334233601887
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,8191,0.012928000340859095
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,16383,0.04497066636880239
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,16383,0.027450665831565857
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,32767,0.08097599943478902
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,127,0.00706666645904382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,32767,0.04562666515509287
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,float16,255,0.007045333584149678
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,65535,0.15403733650843301
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,65535,0.0824480007092158
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,7,0.006666666517655055
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,131071,0.29914132754007977
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,131071,0.1588533322016398
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,63,0.006773333375652631
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,float16,511,0.00867733359336853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,1,0.016864000509182613
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,1,0.014762666076421738
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,3,0.017008000363906223
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,3,0.01482133318980535
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,7,0.014943999548753103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,7,0.01729600007335345
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,511,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,15,0.016890666137139004
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,15,0.01481066644191742
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,31,0.01695999999841054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,31,0.014783999572197596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,63,0.016837333639462788
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,63,0.014805333067973455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,127,0.016895999511082966
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,127,0.015077333897352219
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,255,0.027301333844661713
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,255,0.014757333944241205
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,511,0.045738667249679565
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,1,0.027482666075229645
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,511,0.028042666614055634
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,1,0.02330133318901062
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,1023,0.04677333434422811
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,float16,1023,0.08249600231647491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,3,0.022965334355831146
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,7,0.027258666853109997
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,7,0.022991999983787537
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,3,0.02739199995994568
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,15,0.02738133321205775
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,15,0.023237332701683044
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,31,0.027402666707833607
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,31,0.023141334454218548
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,63,0.027327999472618103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,63,0.023077333966890972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,127,0.029504001140594482
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,127,0.023002666731675465
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,255,0.04550399879614512
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,255,0.027290667096773785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,511,0.08230400085449219
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,511,0.04613333443800608
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,1,0.039461334546407066
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,float16,1023,0.15597866972287497
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,3,0.049072002371152244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,1023,0.08295466502507527
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,3,0.03959999978542328
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,1,0.04797866443792979
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,7,0.04780800143877665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,7,0.03961066653331121
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,15,0.047744000951449074
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,31,0.04875733455022176
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,63,0.0496319979429245
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,63,0.039605334401130676
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,127,0.050069332122802734
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,127,0.040922666589419045
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,15,0.039408000806967415
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,255,0.0806826651096344
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,255,0.04880533119042715
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,31,0.03961600114901861
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,511,0.15260266264279684
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,1,0.02733866622050603
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,1,0.02306666721900304
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,511,0.08330666522185008
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,1023,0.1553920010725657
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,float16,1023,0.29613866408665973
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,3,0.023077333966890972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,3,0.02736533433198929
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,7,0.027461332579453785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,15,0.02740799884001414
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,63,0.027317332724730175
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,7,0.023584000766277313
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,63,0.023189333577950794
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,127,0.02977066735426585
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,127,0.0230880007147789
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,15,0.02317333221435547
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,31,0.027210667729377747
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,31,0.023050665855407715
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,255,0.029418667157491047
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,255,0.045882667104403176
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,1,0.048581331968307495
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,1,0.039520000418027244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,511,0.04770133395989736
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,float16,511,0.08243200182914734
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,3,0.04778666794300079
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,3,0.03972800076007843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,7,0.04870399832725525
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,7,0.03978666663169861
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,15,0.04782933493455251
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,15,0.03947199881076813
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,63,0.04795200129350027
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,63,0.0397173340121905
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,31,0.05004799862702688
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,31,0.039647998909155525
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,127,0.051269332567850746
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,127,0.04053866614898046
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,255,0.08236800134181976
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,255,0.04990399877230326
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,1,0.0724426656961441
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,1,0.08933867017428081
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,float16,511,0.15574933091799417
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,511,0.0839573343594869
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,3,0.0883786678314209
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,3,0.07239466905593872
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,7,0.08785067001978557
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,15,0.08775466680526733
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,7,0.07240533332029979
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,15,0.07240533332029979
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,63,0.08878933389981587
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,31,0.08802666266759236
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,31,0.07252266506354015
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,63,0.0724373310804367
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,1,0.006757333253820737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,3,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,3,0.006789333497484525
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,127,0.09185066819190979
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,7,0.0069973332186539965
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,127,0.07322133580843608
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,7,0.006789333497484525
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,15,0.006613333399097125
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,1,0.0068800002336502075
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,31,0.006624000146985054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,63,0.00679466687142849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,63,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,127,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,127,0.006714666883150737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,255,0.007216000308593114
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,255,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,511,0.008725333337982496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,511,0.006949333474040031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,1023,0.00890666681031386
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,1023,0.008634666601816813
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,2047,0.009082666908701261
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,2047,0.008682666967312494
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,15,0.006720000257094701
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,31,0.007034666836261749
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,4095,0.010965333630641302
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,4095,0.010591999938090643
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,8191,0.014314666390419006
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,8191,0.012847999731699625
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,16383,0.015114666273196539
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,16383,0.029066666960716248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,255,0.15436800320943198
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,32767,0.04756266872088114
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,32767,0.029338667790095013
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,65535,0.08475200335184734
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,65535,0.048432002464930214
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,255,0.08957866827646892
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,1,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,float16,131071,0.15744533141454062
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,3,0.006821333120266597
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,7,0.006949333474040031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,7,0.007045333584149678
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,15,0.006810666372378667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,15,0.006784000123540561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,31,0.006730666384100914
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,31,0.006746666505932808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,63,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,63,0.006805333619316419
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,127,0.006768000001708667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,127,0.006784000123540561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,255,0.006741333131988843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,255,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,511,0.008656000097592672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,511,0.006831999868154526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,1023,0.008901333436369896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,1023,0.008629333227872849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,2047,0.011007999380429586
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,1,0.007386666412154834
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,4095,0.010901333143313726
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,8191,0.026752000053723652
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,8191,0.012938667088747025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,3,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,16383,0.04554666578769684
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,16383,0.02720000098148982
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,32767,0.08226666847864787
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,32767,0.045552000403404236
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,float16,511,0.2965493400891622
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,65535,0.15634133418401083
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,65535,0.08453866839408875
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,1,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,131071,0.30341867605845135
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,131071,0.1620266636212667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,3,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,3,0.006570666407545407
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,7,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,7,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,15,0.006693333387374878
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,15,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,31,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,31,0.006575999781489372
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,float16,4095,0.012789333860079447
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,63,0.00679466687142849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,63,0.006805333619316419
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,127,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,127,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,255,0.007146666447321574
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,511,0.15612799922625223
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,511,0.008698666468262672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,511,0.007887999837597212
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,1023,0.009242666885256767
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,1023,0.008805333326260248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,2047,0.010591999938090643
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,4095,0.027056001126766205
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,4095,0.012736000120639801
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,8191,0.044639999667803444
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,8191,0.027136000494162243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,16383,0.045647998650868736
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,16383,0.08053866525491078
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,32767,0.1528320014476776
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,32767,0.08268266419569652
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,65535,0.29731200138727826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,1,0.00725333330531915
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,2047,0.008832000195980072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,65535,0.15635733803113303
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,1,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,131071,0.08589866757392883
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,255,0.006730666384100914
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,3,0.006741333131988843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,131071,0.5847573280334473
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,7,0.007120000198483467
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,131071,0.3030400077501933
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,7,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,31,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,15,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,63,0.0069973332186539965
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,31,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,63,0.0069919998447100324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,127,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,127,0.006501333167155583
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,255,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,255,0.006698666761318843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,511,0.00873066671192646
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,511,0.007002666592597961
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,1023,0.00884799969693025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,1023,0.008623999853928884
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,2047,0.010762666662534079
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,float16,2047,0.011045332998037338
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,4095,0.012730666746695837
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,8191,0.027077332139015198
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,8191,0.012768000364303589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,16383,0.0454720010360082
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,16383,0.027061333258946735
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,32767,0.08260799944400787
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,1,0.0069919998447100324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,3,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,32767,0.04730666677157084
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,15,0.006704000135262807
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,65535,0.15679466724395752
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,1,0.007370666911204656
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,1,0.008709333216150602
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,3,0.007141333073377609
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,3,0.007109333450595538
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,float16,131071,0.307642658551534
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,2047,0.009029333169261614
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,7,0.007327999919652939
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,4095,0.010741333166758219
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,7,0.007530666887760162
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,31,0.007296000296870868
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,15,0.007327999919652939
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,63,0.007376000285148621
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,31,0.007936000203092894
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,127,0.007135999699433644
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,127,0.007040000210205714
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,65535,0.08468799789746602
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,255,0.006949333474040031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,131071,0.1634880006313324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,255,0.006805333619316419
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,511,0.009039999917149544
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,511,0.008842666943868002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,1023,0.008853333070874214
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,1023,0.009717333440979322
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,2047,0.010597333312034607
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,4095,0.013002666334311167
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,2047,0.011071999867757162
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,8191,0.02735466758410136
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,16383,0.04566933214664459
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,32767,0.15613333384195963
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,65535,0.30212799708048504
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,15,0.007520000139872233
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,65535,0.1567466656366984
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,1,0.00707733320693175
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,63,0.007151999821265538
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,3,0.007087999954819679
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,1,0.00721066693464915
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,3,0.006597333277265231
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,7,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,7,0.006640000268816948
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,15,0.006613333399097125
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,131071,0.5946400165557861
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,31,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,4095,0.027210667729377747
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,63,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,63,0.006746666505932808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,127,0.006682666639486949
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,131071,0.30445865790049237
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,31,0.00655466690659523
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,127,0.006725333631038666
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,16383,0.08230400085449219
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,255,0.00666133314371109
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,511,0.008634666601816813
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,511,0.007098666702707608
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,1023,0.008634666601816813
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,255,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,2047,0.008645333349704742
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,32767,0.08397866288820903
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,2047,0.008821333448092142
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,4095,0.008592000231146812
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,4095,0.008826666822036108
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,8191,0.008992000172535578
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,16383,0.012837332983811697
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,16383,0.012682666381200155
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,32767,0.015178666760524115
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,32767,0.016789333273967106
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,65535,0.02513599892457326
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,65535,0.02309866746266683
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,float16,131071,0.029338667790095013
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,131071,0.02712533374627431
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,1,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,1,0.006517333288987477
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,3,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,3,0.006613333399097125
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,7,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,15,0.007007999966541926
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,15,0.006671999891599019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,31,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,31,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,63,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,63,0.006821333120266597
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,127,0.007061333085099856
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,127,0.00666133314371109
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,255,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,255,0.006613333399097125
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,511,0.008634666601816813
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,511,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,1023,0.008623999853928884
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,1023,0.008901333436369896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,2047,0.008725333337982496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,2047,0.008485333373149237
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,15,0.006511999915043513
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,4095,0.008874666566650072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,4095,0.008943999807039896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,8191,0.009072000160813332
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,8191,0.008767999708652496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,float16,8191,0.045381332437197365
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,16383,0.01292266696691513
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,16383,0.01268799975514412
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,32767,0.017125333348910015
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,65535,0.01887999971707662
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,65535,0.02180800090233485
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,float16,131071,0.039018665750821434
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,1023,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,131071,0.021253332495689392
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,1,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,1,0.006645333642760913
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,3,0.006634666894872983
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,3,0.006688000013430913
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,7,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,7,0.006789333497484525
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,8191,0.008693333094318708
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,15,0.006624000146985054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,15,0.006751999879876773
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,31,0.007040000210205714
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,31,0.006789333497484525
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,63,0.006629333520929019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,63,0.006666666517655055
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,127,0.006714666883150737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,127,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,255,0.0068800002336502075
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,255,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,511,0.008517333616813024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,1023,0.008922666932145754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,2047,0.008538666491707167
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,7,0.006511999915043513
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,4095,0.008933333059151968
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,4095,0.008736000085870424
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,8191,0.008832000195980072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,8191,0.008623999853928884
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,16383,0.013151999562978745
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,16383,0.012725333372751871
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,32767,0.017317333569129307
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,32767,0.016890666137139004
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,65535,0.021477334201335907
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,65535,0.019258666783571243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,float16,131071,0.03885333240032196
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,131071,0.02293866624434789
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,1,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,1,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,3,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,7,0.006949333474040031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,7,0.006618666773041089
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,15,0.006618666773041089
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,15,0.006602666651209195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,31,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,31,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,63,0.0069866664707660675
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,63,0.006560000280539195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,127,0.006831999868154526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,255,0.00697066696981589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,255,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,511,0.0086666668454806
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,511,0.006778666749596596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,32767,0.017504000415404636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,1023,0.00871999996403853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,1023,0.00895999992887179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,2047,0.008714666590094566
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,2047,0.009381333366036415
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,4095,0.009312000125646591
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,4095,0.008687999720374743
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,8191,0.010197333370645842
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,8191,0.008853333070874214
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,511,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,32767,0.01704000060757001
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,1023,0.0085333331177632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,2047,0.00884799969693025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,32767,0.015109332899252573
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,65535,0.03324266771475474
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,65535,0.016858667135238647
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,131071,0.033589333295822144
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,131071,0.05287999908129374
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,1,0.007167999943097432
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,1,0.006629333520929019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,3,0.006954666847983996
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,3,0.006602666651209195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,3,0.006688000013430913
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,7,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,15,0.00701333334048589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,15,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,31,0.006714666883150737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,31,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,63,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,127,0.007930666829148928
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,127,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,255,0.006671999891599019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,255,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,511,0.008623999853928884
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,511,0.007194666812817256
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,16383,0.014762666076421738
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,1023,0.010341333225369453
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,16383,0.013104000439246496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,1023,0.00898133342464765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,2047,0.01102399950226148
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,4095,0.027066667874654133
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,4095,0.013130666067202887
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,8191,0.0455626646677653
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,7,0.006709333509206772
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,16383,0.04578666885693868
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,8191,0.027637332677841187
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,float16,127,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,16383,0.08255999783674876
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,32767,0.1567626694838206
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,32767,0.08282666901747386
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,1,0.008896000062425932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,3,0.008879999940594038
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,7,0.008853333070874214
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,float16,65535,0.30504000186920166
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,65535,0.15876266360282898
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,7,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,15,0.00877333308259646
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,2047,0.010928000013033548
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,31,0.007082666580875714
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,15,0.006640000268816948
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,31,0.008890666688481966
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,63,0.008709333216150602
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,127,0.008602666358153025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,63,0.00706666645904382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,127,0.006671999891599019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,255,0.008698666468262672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,255,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,511,0.008639999975760778
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,511,0.008698666468262672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,1023,0.010757333288590113
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,1023,0.00878399983048439
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,63,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,2047,0.025589334468046825
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,4095,0.044533332188924156
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,4095,0.026757332185904186
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,8191,0.04506133496761322
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,8191,0.08091733356316884
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,16383,0.08330133557319641
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,3,0.007391999786098798
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,1,0.007626666376988093
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,16383,0.15594133734703064
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,32767,0.3031253417332967
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,1,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,32767,0.15664533774058023
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,7,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,3,0.007194666812817256
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,15,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,float16,65535,0.59552534421285
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,31,0.006778666749596596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,3,0.006810666372378667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,63,0.007029333462317784
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,63,0.007807999849319458
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,127,0.006693333387374878
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,127,0.006629333520929019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,31,0.006538666784763336
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,255,0.006789333497484525
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,2047,0.012693333129088083
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,255,0.006634666894872983
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,511,0.008879999940594038
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,511,0.007018666714429855
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,1023,0.008746666833758354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,2047,0.008997333546479544
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,2047,0.008778666456540426
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,1023,0.00884799969693025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,4095,0.008789333204428354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,4095,0.008474666625261307
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,8191,0.009056000038981438
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,8191,0.00867733359336853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,16383,0.014762666076421738
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,16383,0.013167999684810638
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,32767,0.017301333447297413
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,32767,0.01524266724785169
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,65535,0.033146666983763375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,65535,0.017269333203633625
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,float16,131071,0.05170666674772898
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,131071,0.03390933324893316
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,1,0.006805333619316419
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,3,0.007029333462317784
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,3,0.006560000280539195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,7,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,7,0.006741333131988843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,15,0.006949333474040031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,15,0.0069919998447100324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,31,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,31,0.006597333277265231
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,63,0.007184000064929326
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,127,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,127,0.006773333375652631
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,255,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,255,0.006448000048597653
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,511,0.008810666700204214
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,65535,0.3038826584815979
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,7,0.006784000123540561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,511,0.007333333293596904
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,1023,0.008869333192706108
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,2047,0.008949333180983862
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,1023,0.00874133345981439
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,2047,0.008570666735370954
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,4095,0.008714666590094566
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,4095,0.008586666857202848
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,8191,0.012800000607967377
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,8191,0.011002667248249054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,16383,0.014720000326633453
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,32767,0.029706666866938274
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,32767,0.01553600033124288
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,65535,0.048714667558670044
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,65535,0.031130666534105938
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,131071,0.08577066659927368
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,float16,1,0.006671999891599019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,131071,0.04974400003751119
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,1,0.008527999743819237
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,1,0.007007999966541926
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,3,0.008767999708652496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,63,0.007642666498819987
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,3,0.00701333334048589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,7,0.008736000085870424
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,7,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,15,0.007071999832987785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,15,0.008778666456540426
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,15,0.007093333328763644
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,31,0.008634666601816813
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,31,0.006704000135262807
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,63,0.008922666932145754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,63,0.006725333631038666
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,127,0.008687999720374743
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,1,0.0069866664707660675
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,255,0.008586666857202848
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,255,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,511,0.008645333349704742
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,511,0.008650666723648706
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,1023,0.010672000547250112
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,1023,0.008933333059151968
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,2047,0.02698666602373123
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,2047,0.01268799975514412
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,4095,0.045738667249679565
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,8191,0.08222933113574982
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,4095,0.027237333357334137
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,8191,0.04586666822433472
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,1,0.008832000195980072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,16383,0.15637866655985513
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,16383,0.08437866965929668
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,16383,0.012794667234023413
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,127,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,3,0.009029333169261614
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,1,0.009018666421373686
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,7,0.008570666735370954
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,15,0.008767999708652496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,15,0.008976000050703684
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,7,0.00919999989370505
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,3,0.008656000097592672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,31,0.009077333534757296
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,float16,32767,0.30428266525268555
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,31,0.008858666444818178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,63,0.008986666798591614
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,63,0.009029333169261614
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,127,0.008810666700204214
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,127,0.009349333122372627
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,255,0.009050666665037474
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,511,0.011509332805871964
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,511,0.010687999427318573
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,255,0.008805333326260248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,1023,0.026863999664783478
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,32767,0.15803200006484985
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,4095,0.046223998069763184
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,2047,0.02739733209212621
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,2047,0.0447626660267512
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,8191,0.156741331020991
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,8191,0.08444266517957051
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,16383,0.3042400081952413
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,16383,0.15827733278274536
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,1,0.009114666531483332
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,1,0.00879466657837232
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,3,0.009450666606426239
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,1023,0.012837332983811697
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,3,0.009082666908701261
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,4095,0.0821973333756129
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,7,0.009039999917149544
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,7,0.008922666932145754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,15,0.008714666590094566
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,31,0.00895999992887179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,float16,32767,0.6017866532007853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,32767,0.3126773238182068
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,63,0.009050666665037474
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,63,0.00902399979531765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,127,0.008853333070874214
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,127,0.008832000195980072
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,511,0.011514666179815928
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,511,0.010778666784365972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,1023,0.026933332284291584
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,1023,0.013669333110253016
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,15,0.00867733359336853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,255,0.008949333180983862
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,2047,0.04562133550643921
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,2047,0.027280000348885853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,4095,0.08255999783674876
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,8191,0.15691199898719788
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,4095,0.045567999283472695
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,1,0.012890666723251343
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,8191,0.08392533659934998
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,3,0.012661332885424295
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,31,0.008762666955590248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,float16,255,0.00891733355820179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,1,0.010992000500361124
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,7,0.012671999633312225
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,7,0.010735999792814255
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,15,0.01259200026591619
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,3,0.010709332923094431
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,15,0.010714666297038397
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,31,0.01250133290886879
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,31,0.010933333386977514
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,63,0.012469333906968435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,63,0.01098666712641716
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,127,0.012821332861979803
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,127,0.010597333312034607
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,255,0.011333333949247995
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,255,0.010816000401973724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,511,0.027077332139015198
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,511,0.013408000270525614
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,1023,0.045834665497144066
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,1023,0.02736533433198929
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,4095,0.08247466882069905
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,2047,0.08255466818809509
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,2047,0.0454720010360082
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,8191,0.31065066655476886
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,8191,0.15913599729537964
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,1,0.012896000097195307
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,3,0.012773333738247553
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,1,0.010970667004585266
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,float16,4095,0.1572533349196116
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,7,0.010602666685978571
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,31,0.012309333930412928
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,31,0.011007999380429586
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,15,0.012810666114091873
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,7,0.012757333616415659
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,3,0.010928000013033548
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,63,0.012879999975363413
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,63,0.010703999549150467
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,127,0.01232533281048139
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,127,0.010949333508809408
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,255,0.01180800050497055
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,255,0.010746666540702185
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,511,0.02712533374627431
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,15,0.010714666297038397
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,511,0.012981332838535309
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,1023,0.027749332288901012
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,1023,0.04554133117198944
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,1,0.016890666137139004
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,1,0.01492799942692121
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,2047,0.08408533533414204
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,7,0.016943999876578648
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,3,0.01720000058412552
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,3,0.014938666174809137
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,7,0.015119999647140503
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,4095,0.0837653378645579
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,float16,4095,0.15858667095502219
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,15,0.014757333944241205
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,31,0.016890666137139004
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,15,0.01716800034046173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,31,0.01482133318980535
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,63,0.01701333373785019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,2047,0.046629334489504494
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,127,0.01471466695268949
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,255,0.025909334421157837
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,255,0.014757333944241205
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,511,0.04399466514587402
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,511,0.02733866622050603
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,1023,0.08163199822107951
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,1023,0.04585599899291992
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,63,0.01504533365368843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,4095,0.3011786739031474
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,127,0.017210666090250015
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,1,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,7,0.0069440001000960665
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,7,0.006778666749596596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,15,0.006768000001708667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,1,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,15,0.006549333532651265
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,31,0.006842666616042455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,31,0.006714666883150737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,63,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,63,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,127,0.0069973332186539965
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,127,0.006501333167155583
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,255,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,255,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,511,0.008469333251317343
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,float16,2047,0.15569600462913513
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,511,0.006853333363930385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,2047,0.08310399949550629
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,1023,0.008912000184257826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,1023,0.008325333396593729
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,2047,0.009008000294367472
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,4095,0.008650666723648706
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,4095,0.009749333063761393
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,8191,0.012757333616415659
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,8191,0.010703999549150467
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,16383,0.013189333180586496
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,32767,0.030394665896892548
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,32767,0.016917333006858826
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,65535,0.04957333207130432
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,65535,0.03148799886306127
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,3,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,2047,0.009056000038981438
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,131071,0.04980800052483877
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,1,0.007071999832987785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,1,0.006671999891599019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,3,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,3,0.006709333509206772
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,7,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,7,0.007109333450595538
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,15,0.007157333195209503
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,15,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,31,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,31,0.006677333265542984
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,63,0.0068800002336502075
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,63,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,127,0.007055999711155891
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,127,0.006784000123540561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,255,0.007055999711155891
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,255,0.006981333096822103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,511,0.008842666943868002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,511,0.00702400008837382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,1023,0.009061333412925402
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,1023,0.008645333349704742
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,2047,0.009888000165422758
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,2047,0.008736000085870424
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,4095,0.01102399950226148
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,16383,0.015125333021084467
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,4095,0.011002667248249054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,8191,0.012624000509579977
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,8191,0.014277332772811254
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,16383,0.029322666426499683
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,16383,0.015002666662136713
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,32767,0.04776533444722494
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,32767,0.02992533395687739
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,65535,0.08397333820660909
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,65535,0.04783466458320618
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,float16,131071,0.08604266246159871
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,131071,0.08686932921409607
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,float16,131071,0.15708800156911215
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,3,0.00666133314371109
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,1,0.01676799977819125
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,4095,0.1591200033823649
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,1,0.014959999670584997
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,3,0.017103999853134155
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,3,0.01492799942692121
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,15,0.01691199963291486
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,7,0.015114666273196539
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,15,0.014709333578745524
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,7,0.01687466725707054
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,31,0.016778666526079178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,63,0.017130666722853977
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,31,0.01523200049996376
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,63,0.015141333142916361
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,127,0.016927999754746754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,127,0.015087999403476715
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,255,0.02701866626739502
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,255,0.01492799942692121
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,511,0.04554666578769684
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,511,0.027248000105222065
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,1023,0.04643733302752177
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,1,0.027445333699385326
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,1023,0.08245866497357686
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,1,0.02293333411216736
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,float16,2047,0.1553386648495992
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,2047,0.08449600140253703
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,3,0.02749866743882497
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,3,0.023039999107519787
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,7,0.0229120006163915
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,15,0.02313599983851115
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,7,0.027632000545660656
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,15,0.027488000690937042
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,63,0.027109332382678986
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,127,0.02897600084543228
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,127,0.023024000227451324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,255,0.04518933097521464
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,31,0.027269333600997925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,63,0.023039999107519787
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,255,0.028165332973003387
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,31,0.023354666928450268
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,511,0.08057599763075511
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,511,0.045408000548680626
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,1023,0.1552853286266327
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,1023,0.08274133503437042
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,1,0.027488000690937042
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,1,0.023408000667889912
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,float16,2047,0.30144532521565753
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,3,0.023013333479563396
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,15,0.02739199995994568
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,2047,0.15647466977437338
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,15,0.023205332458019257
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,7,0.027077332139015198
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,7,0.023056000471115112
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,31,0.023178666830062866
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,31,0.027232001225153606
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,3,0.027285332481066387
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,63,0.027263998985290527
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,63,0.023077333966890972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,127,0.029792000850041706
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,127,0.023039999107519787
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,255,0.02900800108909607
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,511,0.08303999900817871
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,511,0.04641599953174591
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,1,0.04762133459250132
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,1,0.03989866624275843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,255,0.045423999428749084
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,1023,0.08255466818809509
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,float16,1023,0.1554026703039805
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,3,0.0488319993019104
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,3,0.03976533313592275
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,7,0.047824000318845115
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,7,0.03950933367013931
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,15,0.03977066775163015
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,15,0.047983999053637184
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,63,0.04882133503754934
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,31,0.0487306664387385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,63,0.03944533318281174
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,31,0.039706667264302574
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,127,0.05087999999523163
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,127,0.04155199974775314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,255,0.08195733527342479
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,1,0.006773333375652631
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,3,0.006645333642760913
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,7,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,7,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,1,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,15,0.006954666847983996
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,15,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,31,0.006864000111818314
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,31,0.006751999879876773
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,63,0.0069919998447100324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,63,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,127,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,127,0.006746666505932808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,255,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,3,0.007002666592597961
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,255,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,511,0.008496000121037165
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,511,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,1023,0.008810666700204214
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,1023,0.008522666369875273
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,2047,0.008922666932145754
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,2047,0.008698666468262672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,4095,0.01097600037852923
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,255,0.049360002080599465
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,4095,0.010698666175206503
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,8191,0.014090667168299357
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,8191,0.012810666114091873
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,16383,0.015072000523408255
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,16383,0.029391999046007793
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,32767,0.029338667790095013
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,65535,0.04854933420817057
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,65535,0.0846613347530365
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,1,0.007098666702707608
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,131071,0.15801067153612772
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,131071,0.08629866441090901
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,3,0.006725333631038666
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,7,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,511,0.15398400028546652
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,7,0.006570666407545407
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,15,0.006805333619316419
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,float16,32767,0.04775466521581014
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,31,0.006815999746322632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,31,0.007087999954819679
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,63,0.0068800002336502075
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,63,0.006709333509206772
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,127,0.006645333642760913
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,127,0.006650666395823161
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,255,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,255,0.006949333474040031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,511,0.0074560002734263735
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,15,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,1,0.006714666883150737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,1023,0.008645333349704742
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,2047,0.010698666175206503
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,2047,0.0086666668454806
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,4095,0.012789333860079447
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,4095,0.010768000036478043
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,8191,0.02717333287000656
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,8191,0.01310933381319046
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,16383,0.045514668027559914
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,16383,0.027280000348885853
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,32767,0.08228266735871632
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,32767,0.045925334095954895
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,65535,0.08261333405971527
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,65535,0.15632533033688864
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,float16,1023,0.30057066679000854
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,511,0.008586666857202848
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,1023,0.010714666297038397
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,1,0.006688000013430913
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,131071,0.30421332518259686
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,131071,0.15661866466204324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,3,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,3,0.00696000022192796
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,7,0.006629333520929019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,7,0.006528000036875407
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,15,0.006949333474040031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,15,0.006682666639486949
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,31,0.007029333462317784
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,31,0.00660800002515316
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,63,0.006746666505932808
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,63,0.006768000001708667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,127,0.006666666517655055
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,127,0.006778666749596596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,255,0.007055999711155891
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,1023,0.15679466724395752
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,255,0.006698666761318843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,511,0.008581333483258883
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,511,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,1023,0.008693333094318708
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,1023,0.009312000125646591
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,2047,0.010821333775917688
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,4095,0.012736000120639801
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,4095,0.01080000028014183
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,8191,0.027258666853109997
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,16383,0.027456000447273254
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,8191,0.012778667112191519
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,16383,0.04561600089073181
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,float16,3,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,32767,0.0823466678460439
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,511,0.08285333216190338
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,32767,0.04677333434422811
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,1,0.006773333375652631
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,65535,0.15718400478363037
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,3,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,65535,0.08565333485603333
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,7,0.007071999832987785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,3,0.0074560002734263735
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,7,0.007071999832987785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,15,0.0069973332186539965
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,131071,0.15812800327936807
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,15,0.006741333131988843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,31,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,31,0.006671999891599019
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,1,0.00702400008837382
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,63,0.006682666639486949
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,127,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,255,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,255,0.006778666749596596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,511,0.00854399986565113
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,511,0.006831999868154526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,1023,0.006981333096822103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,2047,0.008789333204428354
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,2047,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,4095,0.008976000050703684
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,4095,0.00884799969693025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,8191,0.009119999905427298
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,8191,0.008687999720374743
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,16383,0.012730666746695837
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,32767,0.015301333119471868
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,32767,0.014869333555301031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,65535,0.02478933334350586
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,65535,0.023024000227451324
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,131071,0.029167999823888142
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,1,0.006949333474040031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,131071,0.0272533322374026
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,1,0.006698666761318843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,1,0.007237333183487256
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,3,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,float16,131071,0.3061013420422872
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,127,0.006821333120266597
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,63,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,7,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,2047,0.009103999783595404
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,31,0.006709333509206772
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,7,0.006480000292261441
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,float16,1023,0.008672000219424566
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,31,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,63,0.006821333120266597
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,63,0.006645333642760913
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,127,0.006688000013430913
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,127,0.006757333253820737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,255,0.006831999868154526
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,255,0.006773333375652631
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,511,0.008650666723648706
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,1023,0.008698666468262672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,1023,0.008527999743819237
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,2047,0.008650666723648706
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,2047,0.008672000219424566
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,4095,0.008687999720374743
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,4095,0.008613333106040955
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,8191,0.008858666444818178
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,8191,0.008698666468262672
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,16383,0.01292266696691513
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,16383,0.012746666868527731
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,32767,0.017162666966517765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,32767,0.016997333616018295
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,65535,0.02145066608985265
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,65535,0.019823999454577763
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,131071,0.03887466589609782
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,131071,0.021850667893886566
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,1,0.006778666749596596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,1,0.00666133314371109
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,3,0.006805333619316419
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,3,0.00690133310854435
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,7,0.006821333120266597
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,7,0.006693333387374878
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,15,0.006677333265542984
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,15,0.006533333410819371
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,31,0.0069333333522081375
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,31,0.007104000076651573
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,511,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,63,0.006800000245372455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,63,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,127,0.006906666482488315
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,16383,0.011087999989589056
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,127,0.006800000245372455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,255,0.007034666836261749
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,255,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,511,0.008885333314538002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,511,0.006837333242098491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,1023,0.009056000038981438
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,2047,0.010847999403874079
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,2047,0.010634666929642359
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,4095,0.027215999861558277
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,4095,0.013066666821638743
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,8191,0.04535466432571411
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,3,0.006682666639486949
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,float16,15,0.006778666749596596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,8191,0.027786667148272198
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,16383,0.04595200220743815
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,15,0.00684799998998642
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,16383,0.08286400139331818
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,32767,0.1567039986451467
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,32767,0.08351999521255493
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,3,0.0068800002336502075
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,3,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,7,0.007247999931375186
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,65535,0.1586079994837443
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,7,0.0069386667261521024
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,1023,0.00895999992887179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,15,0.006495999793211619
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,31,0.00720000018676122
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,15,0.010213333492477735
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,63,0.007247999931375186
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,131071,0.6009173393249512
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,63,0.0074506668994824094
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,127,0.007360000163316727
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,255,0.007402666533986728
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,255,0.007146666447321574
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,1,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,511,0.007141333073377609
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,1023,0.009056000038981438
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,1023,0.007429333403706551
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,2047,0.008863999818762144
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,2047,0.008623999853928884
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,float16,65535,0.3037760059038798
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,4095,0.008992000172535578
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,4095,0.008885333314538002
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,8191,0.00898133342464765
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,16383,0.015029333531856537
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,16383,0.01314666618903478
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,8191,0.008810666700204214
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,32767,0.01695466662446658
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,32767,0.015029333531856537
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,65535,0.03284800052642822
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,65535,0.017071999609470367
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,131071,0.05162666738033295
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,131071,0.033471999069054924
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,1,0.008762666955590248
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,1,0.006821333120266597
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,3,0.00843733362853527
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,1,0.006741333131988843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,511,0.009077333534757296
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,3,0.007055999711155891
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,7,0.0086666668454806
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,7,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,15,0.008661333471536636
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,15,0.006784000123540561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,31,0.00860799973209699
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,31,0.006917333230376244
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,63,0.00850133349498113
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,131071,0.3105386694272359
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,63,0.0068693334857622785
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,127,0.008623999853928884
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,127,0.006741333131988843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,255,0.008639999975760778
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,255,0.006927999978264173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,float16,31,0.007322666545708974
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,511,0.008586666857202848
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,1023,0.010784000158309937
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,1023,0.009119999905427298
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,2047,0.02603733291228612
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,2047,0.01267733300725619
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,8191,0.08286933104197185
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,127,0.0075093333919843035
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,4095,0.02737066646416982
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,4095,0.04466133316357931
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,8191,0.04584533472855886
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,32767,0.15622933705647787
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,16383,0.1562933325767517
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,32767,0.3042293389638265
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,16383,0.0836853285630544
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,1,0.009050666665037474
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,511,0.009125333279371262
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,1,0.008901333436369896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,3,0.00914666677514712
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,3,0.008709333216150602
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,7,0.009050666665037474
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,7,0.009103999783595404
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,float16,65535,0.5994079907735189
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,15,0.00891733355820179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,31,0.009045333291093508
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,65535,0.30636799335479736
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,63,0.008901333436369896
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,127,0.009050666665037474
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,63,0.008613333106040955
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,127,0.008896000062425932
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,255,0.008863999818762144
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,511,0.01184533288081487
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,511,0.010842667271693548
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,1023,0.013045333325862885
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,15,0.008757333581646284
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,255,0.008992000172535578
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,31,0.008639999975760778
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,2047,0.02700799951950709
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,2047,0.0452106644709905
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,8191,0.15819199879964194
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,8191,0.0849120020866394
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,4095,0.08277866741021474
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,4095,0.04548799991607666
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,16383,0.3051466743151347
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,float16,1023,0.02703999976317088
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,16383,0.16088533401489258
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,1,0.010714666297038397
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,1,0.012655999511480331
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,3,0.012293333808581034
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,7,0.010549332946538925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,3,0.010672000547250112
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,15,0.012821332861979803
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,31,0.012917333592971167
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,7,0.0129120002190272
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,31,0.010960000256697336
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,15,0.010746666540702185
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,63,0.012565333396196365
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,63,0.010768000036478043
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,127,0.012773333738247553
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,127,0.010794666906197866
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,255,0.010735999792814255
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,255,0.0129120002190272
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,511,0.027237333357334137
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,1023,0.04569066564242045
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,1023,0.027072000006834667
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,1,0.006698666761318843
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,511,0.014661333213249842
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,8191,0.31041600306828815
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,2047,0.08267733454704285
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,3,0.006725333631038666
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,2047,0.04563199977080027
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,7,0.006778666749596596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,3,0.006800000245372455
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,15,0.006575999781489372
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,31,0.006714666883150737
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,31,0.006501333167155583
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,63,0.006549333532651265
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,63,0.007029333462317784
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,127,0.0069226666043202085
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,127,0.0068853336075941724
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,255,0.006677333265542984
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,255,0.006613333399097125
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,511,0.008602666358153025
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,511,0.006895999734600385
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,1023,0.008687999720374743
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,1023,0.00850133349498113
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,2047,0.008629333227872849
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,2047,0.008576000109314919
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,7,0.00726400005320708
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,float16,4095,0.15803733468055725
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,4095,0.008693333094318708
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,4095,0.08338666955629985
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,4095,0.008821333448092142
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,8191,0.01232533281048139
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,8191,0.010735999792814255
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,1,0.006602666651209195
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,32767,0.016832000265518825
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,32767,0.03126933425664902
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,65535,0.049498667319615684
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,65535,0.03161066770553589
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,131071,0.049509331583976746
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,131071,0.08678932984670003
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,16383,0.01498666654030482
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,16383,0.01314666618903478
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,8191,0.15773333112398782
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,float16,15,0.007173333317041397
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,1,0.01711999997496605
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,1,0.015029333531856537
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,3,0.01482133318980535
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,7,0.017157333592573803
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,7,0.014848000059525171
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,15,0.014890667051076889
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,31,0.016890666137139004
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,3,0.016997333616018295
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,63,0.01714666684468587
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,63,0.014848000059525171
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,127,0.01718933383623759
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,15,0.017055999487638474
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,31,0.014933332800865173
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,127,0.014842666685581207
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,255,0.026000000536441803
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,255,0.014826666563749313
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,511,0.04445866743723551
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,511,0.02718399961789449
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,1023,0.08226666847864787
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,1023,0.04594666759173075
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,2047,0.1557813286781311
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,2047,0.08365333080291748
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,1,0.027119999130566914
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,1,0.02334933231274287
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,4095,0.1595146656036377
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,float16,4095,0.30204800764719647
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,3,0.02314666658639908
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,3,0.027327999472618103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,15,0.027327999472618103
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,15,0.023082666099071503
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,7,0.02733866622050603
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,7,0.02332266668478648
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,63,0.02722666660944621
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,63,0.023077333966890972
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,127,0.029130667448043823
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,31,0.027237333357334137
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,31,0.02313599983851115
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,127,0.023386667172114056
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,255,0.04541866481304169
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,255,0.02918400118748347
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,1,0.006858666737874349
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,3,0.006650666395823161
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,3,0.006570666407545407
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,1,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,7,0.006778666749596596
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,7,0.006762666627764702
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,15,0.006826666494210561
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,15,0.006735999758044879
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,31,0.006949333474040031
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,31,0.006645333642760913
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,63,0.006965333595871925
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,511,0.08211733400821686
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,63,0.006911999856432279
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,127,0.006890666360656421
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,127,0.006874666859706243
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,255,0.006735999758044879
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,511,0.008933333059151968
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,511,0.007093333328763644
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,1023,0.00860799973209699
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,1023,0.00877333308259646
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,2047,0.00891733355820179
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,2047,0.008586666857202848
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,4095,0.010832000523805618
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,511,0.04577066500981649
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,4095,0.010682666053374609
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,8191,0.01379199946920077
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,8191,0.012725333372751871
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,16383,0.014757333944241205
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,32767,0.04746133089065552
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,255,0.006954666847983996
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,32767,0.029343999922275543
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,65535,0.08515733480453491
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,65535,0.04864533245563507
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,131071,0.15798933307329813
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,131071,0.08755733569463094
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,float16,16383,0.02941333254178365
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,1023,0.15475733081499735
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,1023,0.08229866623878479
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,float16,2047,0.3022613326708476
VLLM,0.17.0,NVIDIA B200,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,2047,0.15692800283432007
